Merge ~chad.smith/cloud-init:ubuntu/devel into cloud-init:ubuntu/devel

Proposed by Chad Smith on 2018-03-01
Status: Merged
Merged at revision: 964c869024b422c3966975cc6d23981537084b0a
Proposed branch: ~chad.smith/cloud-init:ubuntu/devel
Merge into: cloud-init:ubuntu/devel
Diff against target: 371 lines (+141/-27)
10 files modified
cloudinit/config/cc_puppet.py (+40/-14)
cloudinit/config/cc_salt_minion.py (+9/-0)
cloudinit/sources/DataSourceGCE.py (+7/-8)
cloudinit/util.py (+7/-2)
debian/changelog (+13/-0)
doc/examples/cloud-config-chef.txt (+2/-2)
tests/cloud_tests/testcases/modules/salt_minion.py (+5/-0)
tests/cloud_tests/testcases/modules/salt_minion.yaml (+5/-0)
tests/unittests/test_datasource/test_gce.py (+19/-1)
tests/unittests/test_util.py (+34/-0)
Reviewer Review Type Date Requested Status
Server Team CI bot continuous-integration Approve on 2018-03-01
Scott Moser 2018-03-01 Pending
Review via email: mp+340252@code.launchpad.net

Description of the change

Sync tip of master to pull in GCE fix for LP: #1752711 and other changes for Bionic

To post a comment you must log in.

PASSED: Continuous integration, rev:964c869024b422c3966975cc6d23981537084b0a
https://jenkins.ubuntu.com/server/job/cloud-init-ci/804/
Executed test runs:
    SUCCESS: Checkout
    SUCCESS: Unit & Style Tests
    SUCCESS: Ubuntu LTS: Build
    SUCCESS: Ubuntu LTS: Integration
    SUCCESS: MAAS Compatability Testing
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/804/rebuild

review: Approve (continuous-integration)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py
2index 28b1d56..57a170f 100644
3--- a/cloudinit/config/cc_puppet.py
4+++ b/cloudinit/config/cc_puppet.py
5@@ -21,6 +21,13 @@ under ``version``, and defaults to ``none``, which selects the latest version
6 in the repos. If the ``puppet`` config key exists in the config archive, this
7 module will attempt to start puppet even if no installation was performed.
8
9+The module also provides keys for configuring the new puppet 4 paths and
10+installing the puppet package from the puppetlabs repositories:
11+https://docs.puppet.com/puppet/4.2/reference/whered_it_go.html
12+The keys are ``package_name``, ``conf_file`` and ``ssl_dir``. If unset, their
13+values will default to ones that work with puppet 3.x and with distributions
14+that ship modified puppet 4.x that uses the old paths.
15+
16 Puppet configuration can be specified under the ``conf`` key. The
17 configuration is specified as a dictionary containing high-level ``<section>``
18 keys and lists of ``<key>=<value>`` pairs within each section. Each section
19@@ -44,6 +51,9 @@ in pem format as a multi-line string (using the ``|`` yaml notation).
20 puppet:
21 install: <true/false>
22 version: <version>
23+ conf_file: '/etc/puppet/puppet.conf'
24+ ssl_dir: '/var/lib/puppet/ssl'
25+ package_name: 'puppet'
26 conf:
27 agent:
28 server: "puppetmaster.example.org"
29@@ -63,9 +73,17 @@ from cloudinit import helpers
30 from cloudinit import util
31
32 PUPPET_CONF_PATH = '/etc/puppet/puppet.conf'
33-PUPPET_SSL_CERT_DIR = '/var/lib/puppet/ssl/certs/'
34 PUPPET_SSL_DIR = '/var/lib/puppet/ssl'
35-PUPPET_SSL_CERT_PATH = '/var/lib/puppet/ssl/certs/ca.pem'
36+PUPPET_PACKAGE_NAME = 'puppet'
37+
38+
39+class PuppetConstants(object):
40+
41+ def __init__(self, puppet_conf_file, puppet_ssl_dir, log):
42+ self.conf_path = puppet_conf_file
43+ self.ssl_dir = puppet_ssl_dir
44+ self.ssl_cert_dir = os.path.join(puppet_ssl_dir, "certs")
45+ self.ssl_cert_path = os.path.join(self.ssl_cert_dir, "ca.pem")
46
47
48 def _autostart_puppet(log):
49@@ -92,22 +110,29 @@ def handle(name, cfg, cloud, log, _args):
50 return
51
52 puppet_cfg = cfg['puppet']
53-
54 # Start by installing the puppet package if necessary...
55 install = util.get_cfg_option_bool(puppet_cfg, 'install', True)
56 version = util.get_cfg_option_str(puppet_cfg, 'version', None)
57+ package_name = util.get_cfg_option_str(
58+ puppet_cfg, 'package_name', PUPPET_PACKAGE_NAME)
59+ conf_file = util.get_cfg_option_str(
60+ puppet_cfg, 'conf_file', PUPPET_CONF_PATH)
61+ ssl_dir = util.get_cfg_option_str(puppet_cfg, 'ssl_dir', PUPPET_SSL_DIR)
62+
63+ p_constants = PuppetConstants(conf_file, ssl_dir, log)
64 if not install and version:
65 log.warn(("Puppet install set false but version supplied,"
66 " doing nothing."))
67 elif install:
68 log.debug(("Attempting to install puppet %s,"),
69 version if version else 'latest')
70- cloud.distro.install_packages(('puppet', version))
71+
72+ cloud.distro.install_packages((package_name, version))
73
74 # ... and then update the puppet configuration
75 if 'conf' in puppet_cfg:
76 # Add all sections from the conf object to puppet.conf
77- contents = util.load_file(PUPPET_CONF_PATH)
78+ contents = util.load_file(p_constants.conf_path)
79 # Create object for reading puppet.conf values
80 puppet_config = helpers.DefaultingConfigParser()
81 # Read puppet.conf values from original file in order to be able to
82@@ -116,19 +141,19 @@ def handle(name, cfg, cloud, log, _args):
83 cleaned_lines = [i.lstrip() for i in contents.splitlines()]
84 cleaned_contents = '\n'.join(cleaned_lines)
85 puppet_config.readfp(StringIO(cleaned_contents),
86- filename=PUPPET_CONF_PATH)
87+ filename=p_constants.conf_path)
88 for (cfg_name, cfg) in puppet_cfg['conf'].items():
89 # Cert configuration is a special case
90 # Dump the puppet master ca certificate in the correct place
91 if cfg_name == 'ca_cert':
92 # Puppet ssl sub-directory isn't created yet
93 # Create it with the proper permissions and ownership
94- util.ensure_dir(PUPPET_SSL_DIR, 0o771)
95- util.chownbyname(PUPPET_SSL_DIR, 'puppet', 'root')
96- util.ensure_dir(PUPPET_SSL_CERT_DIR)
97- util.chownbyname(PUPPET_SSL_CERT_DIR, 'puppet', 'root')
98- util.write_file(PUPPET_SSL_CERT_PATH, cfg)
99- util.chownbyname(PUPPET_SSL_CERT_PATH, 'puppet', 'root')
100+ util.ensure_dir(p_constants.ssl_dir, 0o771)
101+ util.chownbyname(p_constants.ssl_dir, 'puppet', 'root')
102+ util.ensure_dir(p_constants.ssl_cert_dir)
103+ util.chownbyname(p_constants.ssl_cert_dir, 'puppet', 'root')
104+ util.write_file(p_constants.ssl_cert_path, cfg)
105+ util.chownbyname(p_constants.ssl_cert_path, 'puppet', 'root')
106 else:
107 # Iterate through the config items, we'll use ConfigParser.set
108 # to overwrite or create new items as needed
109@@ -144,8 +169,9 @@ def handle(name, cfg, cloud, log, _args):
110 puppet_config.set(cfg_name, o, v)
111 # We got all our config as wanted we'll rename
112 # the previous puppet.conf and create our new one
113- util.rename(PUPPET_CONF_PATH, "%s.old" % (PUPPET_CONF_PATH))
114- util.write_file(PUPPET_CONF_PATH, puppet_config.stringify())
115+ util.rename(p_constants.conf_path, "%s.old"
116+ % (p_constants.conf_path))
117+ util.write_file(p_constants.conf_path, puppet_config.stringify())
118
119 # Set it up so it autostarts
120 _autostart_puppet(log)
121diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py
122index 2b38837..5112a34 100644
123--- a/cloudinit/config/cc_salt_minion.py
124+++ b/cloudinit/config/cc_salt_minion.py
125@@ -25,6 +25,9 @@ specified with ``public_key`` and ``private_key`` respectively.
126 salt_minion:
127 conf:
128 master: salt.example.com
129+ grains:
130+ role:
131+ - web
132 public_key: |
133 ------BEGIN PUBLIC KEY-------
134 <key data>
135@@ -65,6 +68,12 @@ def handle(name, cfg, cloud, log, _args):
136 minion_data = util.yaml_dumps(salt_cfg.get('conf'))
137 util.write_file(minion_config, minion_data)
138
139+ if 'grains' in salt_cfg:
140+ # add grains to /etc/salt/grains
141+ grains_config = os.path.join(config_dir, 'grains')
142+ grains_data = util.yaml_dumps(salt_cfg.get('grains'))
143+ util.write_file(grains_config, grains_data)
144+
145 # ... copy the key pair if specified
146 if 'public_key' in salt_cfg and 'private_key' in salt_cfg:
147 if os.path.isdir("/etc/salt/pki/minion"):
148diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py
149index 2da34a9..bebc991 100644
150--- a/cloudinit/sources/DataSourceGCE.py
151+++ b/cloudinit/sources/DataSourceGCE.py
152@@ -213,16 +213,15 @@ def read_md(address=None, platform_check=True):
153 if md['availability-zone']:
154 md['availability-zone'] = md['availability-zone'].split('/')[-1]
155
156- encoding = instance_data.get('user-data-encoding')
157- if encoding:
158+ if 'user-data' in instance_data:
159+ # instance_data was json, so values are all utf-8 strings.
160+ ud = instance_data['user-data'].encode("utf-8")
161+ encoding = instance_data.get('user-data-encoding')
162 if encoding == 'base64':
163- md['user-data'] = b64decode(instance_data.get('user-data'))
164- else:
165+ ud = b64decode(ud)
166+ elif encoding:
167 LOG.warning('unknown user-data-encoding: %s, ignoring', encoding)
168-
169- if 'user-data' in md:
170- ret['user-data'] = md['user-data']
171- del md['user-data']
172+ ret['user-data'] = ud
173
174 ret['meta-data'] = md
175 ret['success'] = True
176diff --git a/cloudinit/util.py b/cloudinit/util.py
177index 338fb97..02dc2ce 100644
178--- a/cloudinit/util.py
179+++ b/cloudinit/util.py
180@@ -1746,7 +1746,7 @@ def chmod(path, mode):
181 def write_file(filename, content, mode=0o644, omode="wb", copy_mode=False):
182 """
183 Writes a file with the given content and sets the file mode as specified.
184- Resotres the SELinux context if possible.
185+ Restores the SELinux context if possible.
186
187 @param filename: The full path of the file to write.
188 @param content: The content to write to the file.
189@@ -1865,8 +1865,13 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False,
190 if not isinstance(data, bytes):
191 data = data.encode()
192
193+ # Popen converts entries in the arguments array from non-bytes to bytes.
194+ # When locale is unset it may use ascii for that encoding which can
195+ # cause UnicodeDecodeErrors. (LP: #1751051)
196+ bytes_args = [x if isinstance(x, six.binary_type) else x.encode("utf-8")
197+ for x in args]
198 try:
199- sp = subprocess.Popen(args, stdout=stdout,
200+ sp = subprocess.Popen(bytes_args, stdout=stdout,
201 stderr=stderr, stdin=stdin,
202 env=env, shell=shell)
203 (out, err) = sp.communicate(data)
204diff --git a/debian/changelog b/debian/changelog
205index 2552bb6..27dba2c 100644
206--- a/debian/changelog
207+++ b/debian/changelog
208@@ -1,3 +1,16 @@
209+cloud-init (18.1-5-g40e77380-0ubuntu1) bionic; urgency=medium
210+
211+ * New upstream snapshot.
212+ - GCE: fix reading of user-data that is not base64 encoded. (LP: #1752711)
213+ - doc: fix chef install from apt packages example in RTD.
214+ - Implement puppet 4 support [Romanos Skiadas] (LP: #1446804)
215+ - subp: Fix subp usage with non-ascii characters when no system locale.
216+ (LP: #1751051)
217+ - salt: configure grains in grains file rather than in minion config.
218+ [Daniel Wallace]
219+
220+ -- Chad Smith <chad.smith@canonical.com> Thu, 01 Mar 2018 15:47:04 -0700
221+
222 cloud-init (18.1-0ubuntu1) bionic; urgency=medium
223
224 * New upstream snapshot.
225diff --git a/doc/examples/cloud-config-chef.txt b/doc/examples/cloud-config-chef.txt
226index 58d5fdc..defc5a5 100644
227--- a/doc/examples/cloud-config-chef.txt
228+++ b/doc/examples/cloud-config-chef.txt
229@@ -12,8 +12,8 @@
230
231 # Key from https://packages.chef.io/chef.asc
232 apt:
233- source1:
234- source: "deb http://packages.chef.io/repos/apt/stable $RELEASE main"
235+ sources:
236+ source1: "deb http://packages.chef.io/repos/apt/stable $RELEASE main"
237 key: |
238 -----BEGIN PGP PUBLIC KEY BLOCK-----
239 Version: GnuPG v1.4.12 (Darwin)
240diff --git a/tests/cloud_tests/testcases/modules/salt_minion.py b/tests/cloud_tests/testcases/modules/salt_minion.py
241index c697db2..f13b48a 100644
242--- a/tests/cloud_tests/testcases/modules/salt_minion.py
243+++ b/tests/cloud_tests/testcases/modules/salt_minion.py
244@@ -26,4 +26,9 @@ class Test(base.CloudTestCase):
245 self.assertIn('<key data>', out)
246 self.assertIn('------END PUBLIC KEY-------', out)
247
248+ def test_grains(self):
249+ """Test master value in config."""
250+ out = self.get_data_file('grains')
251+ self.assertIn('role: web', out)
252+
253 # vi: ts=4 expandtab
254diff --git a/tests/cloud_tests/testcases/modules/salt_minion.yaml b/tests/cloud_tests/testcases/modules/salt_minion.yaml
255index f20d24f..ab0e05b 100644
256--- a/tests/cloud_tests/testcases/modules/salt_minion.yaml
257+++ b/tests/cloud_tests/testcases/modules/salt_minion.yaml
258@@ -17,6 +17,8 @@ cloud_config: |
259 ------BEGIN PRIVATE KEY------
260 <key data>
261 ------END PRIVATE KEY-------
262+ grains:
263+ role: web
264 collect_scripts:
265 minion: |
266 #!/bin/bash
267@@ -30,5 +32,8 @@ collect_scripts:
268 minion.pub: |
269 #!/bin/bash
270 cat /etc/salt/pki/minion/minion.pub
271+ grains: |
272+ #!/bin/bash
273+ cat /etc/salt/grains
274
275 # vi: ts=4 expandtab
276diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py
277index f77c2c4..eb3cec4 100644
278--- a/tests/unittests/test_datasource/test_gce.py
279+++ b/tests/unittests/test_datasource/test_gce.py
280@@ -38,11 +38,20 @@ GCE_META_ENCODING = {
281 'instance/hostname': 'server.project-baz.local',
282 'instance/zone': 'baz/bang',
283 'instance/attributes': {
284- 'user-data': b64encode(b'/bin/echo baz\n').decode('utf-8'),
285+ 'user-data': b64encode(b'#!/bin/echo baz\n').decode('utf-8'),
286 'user-data-encoding': 'base64',
287 }
288 }
289
290+GCE_USER_DATA_TEXT = {
291+ 'instance/id': '12345',
292+ 'instance/hostname': 'server.project-baz.local',
293+ 'instance/zone': 'baz/bang',
294+ 'instance/attributes': {
295+ 'user-data': '#!/bin/sh\necho hi mom\ntouch /run/up-now\n',
296+ }
297+}
298+
299 HEADERS = {'Metadata-Flavor': 'Google'}
300 MD_URL_RE = re.compile(
301 r'http://metadata.google.internal/computeMetadata/v1/.*')
302@@ -135,7 +144,16 @@ class TestDataSourceGCE(test_helpers.HttprettyTestCase):
303 shostname = GCE_META_PARTIAL.get('instance/hostname').split('.')[0]
304 self.assertEqual(shostname, self.ds.get_hostname())
305
306+ def test_userdata_no_encoding(self):
307+ """check that user-data is read."""
308+ _set_mock_metadata(GCE_USER_DATA_TEXT)
309+ self.ds.get_data()
310+ self.assertEqual(
311+ GCE_USER_DATA_TEXT['instance/attributes']['user-data'].encode(),
312+ self.ds.get_userdata_raw())
313+
314 def test_metadata_encoding(self):
315+ """user-data is base64 encoded if user-data-encoding is 'base64'."""
316 _set_mock_metadata(GCE_META_ENCODING)
317 self.ds.get_data()
318
319diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py
320index 4a92e74..89ae40f 100644
321--- a/tests/unittests/test_util.py
322+++ b/tests/unittests/test_util.py
323@@ -8,7 +8,9 @@ import shutil
324 import stat
325 import tempfile
326
327+import json
328 import six
329+import sys
330 import yaml
331
332 from cloudinit import importer, util
333@@ -733,6 +735,38 @@ class TestSubp(helpers.CiTestCase):
334 self.assertEqual("/target/my/path/",
335 util.target_path("/target/", "///my/path/"))
336
337+ def test_c_lang_can_take_utf8_args(self):
338+ """Independent of system LC_CTYPE, args can contain utf-8 strings.
339+
340+ When python starts up, its default encoding gets set based on
341+ the value of LC_CTYPE. If no system locale is set, the default
342+ encoding for both python2 and python3 in some paths will end up
343+ being ascii.
344+
345+ Attempts to use setlocale or patching (or changing) os.environ
346+ in the current environment seem to not be effective.
347+
348+ This test starts up a python with LC_CTYPE set to C so that
349+ the default encoding will be set to ascii. In such an environment
350+ Popen(['command', 'non-ascii-arg']) would cause a UnicodeDecodeError.
351+ """
352+ python_prog = '\n'.join([
353+ 'import json, sys',
354+ 'from cloudinit.util import subp',
355+ 'data = sys.stdin.read()',
356+ 'cmd = json.loads(data)',
357+ 'subp(cmd, capture=False)',
358+ ''])
359+ cmd = [BASH, '-c', 'echo -n "$@"', '--',
360+ self.utf8_valid.decode("utf-8")]
361+ python_subp = [sys.executable, '-c', python_prog]
362+
363+ out, _err = util.subp(
364+ python_subp, update_env={'LC_CTYPE': 'C'},
365+ data=json.dumps(cmd).encode("utf-8"),
366+ decode=False)
367+ self.assertEqual(self.utf8_valid, out)
368+
369
370 class TestEncode(helpers.TestCase):
371 """Test the encoding functions"""

Subscribers

People subscribed via source and target branches