Merge ~chad.smith/cloud-init:ubuntu/artful into cloud-init:ubuntu/artful

Proposed by Chad Smith on 2017-10-23
Status: Merged
Merged at revision: a9968540aa273bee10b0ca040133c01aa3459793
Proposed branch: ~chad.smith/cloud-init:ubuntu/artful
Merge into: cloud-init:ubuntu/artful
Diff against target: 848 lines (+260/-107)
21 files modified
cloudinit/config/cc_lxd.py (+1/-1)
cloudinit/config/cc_ntp.py (+3/-1)
cloudinit/config/cc_resizefs.py (+13/-30)
cloudinit/config/cc_users_groups.py (+2/-1)
cloudinit/config/schema.py (+1/-1)
debian/changelog (+16/-0)
doc/examples/cloud-config-user-groups.txt (+3/-3)
tests/cloud_tests/testcases/__init__.py (+7/-0)
tests/cloud_tests/testcases/base.py (+8/-4)
tests/cloud_tests/testcases/examples/including_user_groups.py (+6/-0)
tests/cloud_tests/testcases/examples/including_user_groups.yaml (+5/-2)
tests/cloud_tests/testcases/main/command_output_simple.py (+16/-0)
tests/cloud_tests/testcases/modules/ntp.yaml (+2/-2)
tests/cloud_tests/testcases/modules/user_groups.py (+6/-0)
tests/cloud_tests/testcases/modules/user_groups.yaml (+5/-2)
tests/unittests/test_handler/test_handler_lxd.py (+8/-8)
tests/unittests/test_handler/test_handler_ntp.py (+12/-11)
tests/unittests/test_handler/test_handler_resizefs.py (+57/-34)
tests/unittests/test_handler/test_schema.py (+36/-1)
tools/read-dependencies (+36/-5)
tools/run-centos (+17/-1)
Reviewer Review Type Date Requested Status
Server Team CI bot continuous-integration Approve on 2017-10-23
Scott Moser 2017-10-23 Pending
Review via email: mp+332673@code.launchpad.net

Description of the change

Upstream snapshot into Artful for SRU

To post a comment you must log in.

PASSED: Continuous integration, rev:a9968540aa273bee10b0ca040133c01aa3459793
https://jenkins.ubuntu.com/server/job/cloud-init-ci/434/
Executed test runs:
    SUCCESS: Checkout
    SUCCESS: Unit & Style Tests
    SUCCESS: Ubuntu LTS: Build
    SUCCESS: Ubuntu LTS: Integration
    SUCCESS: MAAS Compatability Testing
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/434/rebuild

review: Approve (continuous-integration)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py
2index e6262f8..09374d2 100644
3--- a/cloudinit/config/cc_lxd.py
4+++ b/cloudinit/config/cc_lxd.py
5@@ -72,7 +72,7 @@ def handle(name, cfg, cloud, log, args):
6 type(init_cfg))
7 init_cfg = {}
8
9- bridge_cfg = lxd_cfg.get('bridge')
10+ bridge_cfg = lxd_cfg.get('bridge', {})
11 if not isinstance(bridge_cfg, dict):
12 log.warn("lxd/bridge config must be a dictionary. found a '%s'",
13 type(bridge_cfg))
14diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py
15index 15ae1ec..d43d060 100644
16--- a/cloudinit/config/cc_ntp.py
17+++ b/cloudinit/config/cc_ntp.py
18@@ -100,7 +100,9 @@ def handle(name, cfg, cloud, log, _args):
19 LOG.debug(
20 "Skipping module named %s, not present or disabled by cfg", name)
21 return
22- ntp_cfg = cfg.get('ntp', {})
23+ ntp_cfg = cfg['ntp']
24+ if ntp_cfg is None:
25+ ntp_cfg = {} # Allow empty config which will install the package
26
27 # TODO drop this when validate_cloudconfig_schema is strict=True
28 if not isinstance(ntp_cfg, (dict)):
29diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py
30index f774baa..0d282e6 100644
31--- a/cloudinit/config/cc_resizefs.py
32+++ b/cloudinit/config/cc_resizefs.py
33@@ -145,25 +145,6 @@ RESIZE_FS_PRECHECK_CMDS = {
34 }
35
36
37-def rootdev_from_cmdline(cmdline):
38- found = None
39- for tok in cmdline.split():
40- if tok.startswith("root="):
41- found = tok[5:]
42- break
43- if found is None:
44- return None
45-
46- if found.startswith("/dev/"):
47- return found
48- if found.startswith("LABEL="):
49- return "/dev/disk/by-label/" + found[len("LABEL="):]
50- if found.startswith("UUID="):
51- return "/dev/disk/by-uuid/" + found[len("UUID="):]
52-
53- return "/dev/" + found
54-
55-
56 def can_skip_resize(fs_type, resize_what, devpth):
57 fstype_lc = fs_type.lower()
58 for i, func in RESIZE_FS_PRECHECK_CMDS.items():
59@@ -172,14 +153,15 @@ def can_skip_resize(fs_type, resize_what, devpth):
60 return False
61
62
63-def is_device_path_writable_block(devpath, info, log):
64- """Return True if devpath is a writable block device.
65+def maybe_get_writable_device_path(devpath, info, log):
66+ """Return updated devpath if the devpath is a writable block device.
67
68- @param devpath: Path to the root device we want to resize.
69+ @param devpath: Requested path to the root device we want to resize.
70 @param info: String representing information about the requested device.
71 @param log: Logger to which logs will be added upon error.
72
73- @returns Boolean True if block device is writable
74+ @returns devpath or updated devpath per kernel commandline if the device
75+ path is a writable block device, returns None otherwise.
76 """
77 container = util.is_container()
78
79@@ -189,12 +171,12 @@ def is_device_path_writable_block(devpath, info, log):
80 devpath = util.rootdev_from_cmdline(util.get_cmdline())
81 if devpath is None:
82 log.warn("Unable to find device '/dev/root'")
83- return False
84+ return None
85 log.debug("Converted /dev/root to '%s' per kernel cmdline", devpath)
86
87 if devpath == 'overlayroot':
88 log.debug("Not attempting to resize devpath '%s': %s", devpath, info)
89- return False
90+ return None
91
92 try:
93 statret = os.stat(devpath)
94@@ -207,7 +189,7 @@ def is_device_path_writable_block(devpath, info, log):
95 devpath, info)
96 else:
97 raise exc
98- return False
99+ return None
100
101 if not stat.S_ISBLK(statret.st_mode) and not stat.S_ISCHR(statret.st_mode):
102 if container:
103@@ -216,8 +198,8 @@ def is_device_path_writable_block(devpath, info, log):
104 else:
105 log.warn("device '%s' not a block device. cannot resize: %s" %
106 (devpath, info))
107- return False
108- return True
109+ return None
110+ return devpath # The writable block devpath
111
112
113 def handle(name, cfg, _cloud, log, args):
114@@ -242,8 +224,9 @@ def handle(name, cfg, _cloud, log, args):
115 info = "dev=%s mnt_point=%s path=%s" % (devpth, mount_point, resize_what)
116 log.debug("resize_info: %s" % info)
117
118- if not is_device_path_writable_block(devpth, info, log):
119- return
120+ devpth = maybe_get_writable_device_path(devpth, info, log)
121+ if not devpth:
122+ return # devpath was not a writable block device
123
124 resizer = None
125 if can_skip_resize(fs_type, resize_what, devpth):
126diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py
127index b80d1d3..f363000 100644
128--- a/cloudinit/config/cc_users_groups.py
129+++ b/cloudinit/config/cc_users_groups.py
130@@ -15,7 +15,8 @@ options, see the ``Including users and groups`` config example.
131 Groups to add to the system can be specified as a list under the ``groups``
132 key. Each entry in the list should either contain a the group name as a string,
133 or a dictionary with the group name as the key and a list of users who should
134-be members of the group as the value.
135+be members of the group as the value. **Note**: Groups are added before users,
136+so any users in a group list must already exist on the system.
137
138 The ``users`` config key takes a list of users to configure. The first entry in
139 this list is used as the default user for the system. To preserve the standard
140diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py
141index bb291ff..ca7d0d5 100644
142--- a/cloudinit/config/schema.py
143+++ b/cloudinit/config/schema.py
144@@ -74,7 +74,7 @@ def validate_cloudconfig_schema(config, schema, strict=False):
145 try:
146 from jsonschema import Draft4Validator, FormatChecker
147 except ImportError:
148- logging.warning(
149+ logging.debug(
150 'Ignoring schema validation. python-jsonschema is not present')
151 return
152 validator = Draft4Validator(schema, format_checker=FormatChecker())
153diff --git a/debian/changelog b/debian/changelog
154index 26d1d45..5f0c1ce 100644
155--- a/debian/changelog
156+++ b/debian/changelog
157@@ -1,3 +1,19 @@
158+cloud-init (17.1-25-g17a15f9e-0ubuntu1~17.10.1) artful-proposed; urgency=medium
159+
160+ * New upstream snapshot.
161+ - resizefs: Fix regression when system booted with root=PARTUUID=
162+ (LP: #1725067)
163+ - tools: make yum package installation more reliable
164+ - citest: fix remaining warnings raised by integration tests.
165+ - citest: show the class actual class name in results.
166+ - ntp: fix config module schema to allow empty ntp config
167+ (LP: #1724951)
168+ - tools: disable fastestmirror if using proxy [Joshua Powers]
169+ - schema: Log debug instead of warning when jsonschema is not available.
170+ (LP: #1724354)
171+
172+ -- Chad Smith <chad.smith@canonical.com> Mon, 23 Oct 2017 15:07:35 -0600
173+
174 cloud-init (17.1-18-gd4f70470-0ubuntu1) artful; urgency=medium
175
176 * New upstream snapshot.
177diff --git a/doc/examples/cloud-config-user-groups.txt b/doc/examples/cloud-config-user-groups.txt
178index 9c5202f..0554d1f 100644
179--- a/doc/examples/cloud-config-user-groups.txt
180+++ b/doc/examples/cloud-config-user-groups.txt
181@@ -1,8 +1,8 @@
182 # Add groups to the system
183-# The following example adds the ubuntu group with members foo and bar and
184-# the group cloud-users.
185+# The following example adds the ubuntu group with members 'root' and 'sys'
186+# and the empty group cloud-users.
187 groups:
188- - ubuntu: [foo,bar]
189+ - ubuntu: [root,sys]
190 - cloud-users
191
192 # Add users to the system. Users are added after groups are added.
193diff --git a/tests/cloud_tests/testcases/__init__.py b/tests/cloud_tests/testcases/__init__.py
194index 47217ce..a29a092 100644
195--- a/tests/cloud_tests/testcases/__init__.py
196+++ b/tests/cloud_tests/testcases/__init__.py
197@@ -5,6 +5,7 @@
198 import importlib
199 import inspect
200 import unittest
201+from unittest.util import strclass
202
203 from tests.cloud_tests import config
204 from tests.cloud_tests.testcases.base import CloudTestCase as base_test
205@@ -37,6 +38,12 @@ def get_suite(test_name, data, conf):
206
207 class tmp(test_class):
208
209+ _realclass = test_class
210+
211+ def __str__(self):
212+ return "%s (%s)" % (self._testMethodName,
213+ strclass(self._realclass))
214+
215 @classmethod
216 def setUpClass(cls):
217 cls.data = data
218diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py
219index bb545ab..1706f59 100644
220--- a/tests/cloud_tests/testcases/base.py
221+++ b/tests/cloud_tests/testcases/base.py
222@@ -16,10 +16,6 @@ class CloudTestCase(unittest.TestCase):
223 conf = None
224 _cloud_config = None
225
226- def shortDescription(self):
227- """Prevent nose from using docstrings."""
228- return None
229-
230 @property
231 def cloud_config(self):
232 """Get the cloud-config used by the test."""
233@@ -72,6 +68,14 @@ class CloudTestCase(unittest.TestCase):
234 result = self.get_status_data(self.get_data_file('result.json'))
235 self.assertEqual(len(result['errors']), 0)
236
237+ def test_no_warnings_in_log(self):
238+ """Warnings should not be found in the log."""
239+ self.assertEqual(
240+ [],
241+ [l for l in self.get_data_file('cloud-init.log').splitlines()
242+ if 'WARN' in l],
243+ msg="'WARN' found inside cloud-init.log")
244+
245
246 class PasswordListTest(CloudTestCase):
247 """Base password test case class."""
248diff --git a/tests/cloud_tests/testcases/examples/including_user_groups.py b/tests/cloud_tests/testcases/examples/including_user_groups.py
249index 67af527..93b7a82 100644
250--- a/tests/cloud_tests/testcases/examples/including_user_groups.py
251+++ b/tests/cloud_tests/testcases/examples/including_user_groups.py
252@@ -40,4 +40,10 @@ class TestUserGroups(base.CloudTestCase):
253 out = self.get_data_file('user_cloudy')
254 self.assertRegex(out, r'cloudy:x:[0-9]{3,4}:')
255
256+ def test_user_root_in_secret(self):
257+ """Test root user is in 'secret' group."""
258+ user, _, groups = self.get_data_file('root_groups').partition(":")
259+ self.assertIn("secret", groups.split(),
260+ msg="User root is not in group 'secret'")
261+
262 # vi: ts=4 expandtab
263diff --git a/tests/cloud_tests/testcases/examples/including_user_groups.yaml b/tests/cloud_tests/testcases/examples/including_user_groups.yaml
264index 0aa7ad2..469d03c 100644
265--- a/tests/cloud_tests/testcases/examples/including_user_groups.yaml
266+++ b/tests/cloud_tests/testcases/examples/including_user_groups.yaml
267@@ -8,7 +8,7 @@ cloud_config: |
268 #cloud-config
269 # Add groups to the system
270 groups:
271- - secret: [foobar,barfoo]
272+ - secret: [root]
273 - cloud-users
274
275 # Add users to the system. Users are added after groups are added.
276@@ -24,7 +24,7 @@ cloud_config: |
277 - name: barfoo
278 gecos: Bar B. Foo
279 sudo: ALL=(ALL) NOPASSWD:ALL
280- groups: cloud-users
281+ groups: [cloud-users, secret]
282 lock_passwd: true
283 - name: cloudy
284 gecos: Magic Cloud App Daemon User
285@@ -49,5 +49,8 @@ collect_scripts:
286 user_cloudy: |
287 #!/bin/bash
288 getent passwd cloudy
289+ root_groups: |
290+ #!/bin/bash
291+ groups root
292
293 # vi: ts=4 expandtab
294diff --git a/tests/cloud_tests/testcases/main/command_output_simple.py b/tests/cloud_tests/testcases/main/command_output_simple.py
295index fe4c767..857881c 100644
296--- a/tests/cloud_tests/testcases/main/command_output_simple.py
297+++ b/tests/cloud_tests/testcases/main/command_output_simple.py
298@@ -15,4 +15,20 @@ class TestCommandOutputSimple(base.CloudTestCase):
299 data.splitlines()[-1].strip())
300 # TODO: need to test that all stages redirected here
301
302+ def test_no_warnings_in_log(self):
303+ """Warnings should not be found in the log.
304+
305+ This class redirected stderr and stdout, so it expects to find
306+ a warning in cloud-init.log to that effect."""
307+ redirect_msg = 'Stdout, stderr changing to'
308+ warnings = [
309+ l for l in self.get_data_file('cloud-init.log').splitlines()
310+ if 'WARN' in l]
311+ self.assertEqual(
312+ [], [w for w in warnings if redirect_msg not in w],
313+ msg="'WARN' found inside cloud-init.log")
314+ self.assertEqual(
315+ 1, len(warnings),
316+ msg="Did not find %s in cloud-init.log" % redirect_msg)
317+
318 # vi: ts=4 expandtab
319diff --git a/tests/cloud_tests/testcases/modules/ntp.yaml b/tests/cloud_tests/testcases/modules/ntp.yaml
320index fbef431..2530d72 100644
321--- a/tests/cloud_tests/testcases/modules/ntp.yaml
322+++ b/tests/cloud_tests/testcases/modules/ntp.yaml
323@@ -4,8 +4,8 @@
324 cloud_config: |
325 #cloud-config
326 ntp:
327- pools: {}
328- servers: {}
329+ pools: []
330+ servers: []
331 collect_scripts:
332 ntp_installed: |
333 #!/bin/bash
334diff --git a/tests/cloud_tests/testcases/modules/user_groups.py b/tests/cloud_tests/testcases/modules/user_groups.py
335index 67af527..93b7a82 100644
336--- a/tests/cloud_tests/testcases/modules/user_groups.py
337+++ b/tests/cloud_tests/testcases/modules/user_groups.py
338@@ -40,4 +40,10 @@ class TestUserGroups(base.CloudTestCase):
339 out = self.get_data_file('user_cloudy')
340 self.assertRegex(out, r'cloudy:x:[0-9]{3,4}:')
341
342+ def test_user_root_in_secret(self):
343+ """Test root user is in 'secret' group."""
344+ user, _, groups = self.get_data_file('root_groups').partition(":")
345+ self.assertIn("secret", groups.split(),
346+ msg="User root is not in group 'secret'")
347+
348 # vi: ts=4 expandtab
349diff --git a/tests/cloud_tests/testcases/modules/user_groups.yaml b/tests/cloud_tests/testcases/modules/user_groups.yaml
350index 71cc9da..22b5d70 100644
351--- a/tests/cloud_tests/testcases/modules/user_groups.yaml
352+++ b/tests/cloud_tests/testcases/modules/user_groups.yaml
353@@ -7,7 +7,7 @@ cloud_config: |
354 #cloud-config
355 # Add groups to the system
356 groups:
357- - secret: [foobar,barfoo]
358+ - secret: [root]
359 - cloud-users
360
361 # Add users to the system. Users are added after groups are added.
362@@ -23,7 +23,7 @@ cloud_config: |
363 - name: barfoo
364 gecos: Bar B. Foo
365 sudo: ALL=(ALL) NOPASSWD:ALL
366- groups: cloud-users
367+ groups: [cloud-users, secret]
368 lock_passwd: true
369 - name: cloudy
370 gecos: Magic Cloud App Daemon User
371@@ -48,5 +48,8 @@ collect_scripts:
372 user_cloudy: |
373 #!/bin/bash
374 getent passwd cloudy
375+ root_groups: |
376+ #!/bin/bash
377+ groups root
378
379 # vi: ts=4 expandtab
380diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py
381index f132a77..e0d9ab6 100644
382--- a/tests/unittests/test_handler/test_handler_lxd.py
383+++ b/tests/unittests/test_handler/test_handler_lxd.py
384@@ -5,17 +5,16 @@ from cloudinit.sources import DataSourceNoCloud
385 from cloudinit import (distros, helpers, cloud)
386 from cloudinit.tests import helpers as t_help
387
388-import logging
389-
390 try:
391 from unittest import mock
392 except ImportError:
393 import mock
394
395-LOG = logging.getLogger(__name__)
396
397+class TestLxd(t_help.CiTestCase):
398+
399+ with_logs = True
400
401-class TestLxd(t_help.TestCase):
402 lxd_cfg = {
403 'lxd': {
404 'init': {
405@@ -41,7 +40,7 @@ class TestLxd(t_help.TestCase):
406 def test_lxd_init(self, mock_util):
407 cc = self._get_cloud('ubuntu')
408 mock_util.which.return_value = True
409- cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, LOG, [])
410+ cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, self.logger, [])
411 self.assertTrue(mock_util.which.called)
412 init_call = mock_util.subp.call_args_list[0][0][0]
413 self.assertEqual(init_call,
414@@ -55,7 +54,8 @@ class TestLxd(t_help.TestCase):
415 cc = self._get_cloud('ubuntu')
416 cc.distro = mock.MagicMock()
417 mock_util.which.return_value = None
418- cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, LOG, [])
419+ cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, self.logger, [])
420+ self.assertNotIn('WARN', self.logs.getvalue())
421 self.assertTrue(cc.distro.install_packages.called)
422 install_pkg = cc.distro.install_packages.call_args_list[0][0][0]
423 self.assertEqual(sorted(install_pkg), ['lxd', 'zfs'])
424@@ -64,7 +64,7 @@ class TestLxd(t_help.TestCase):
425 def test_no_init_does_nothing(self, mock_util):
426 cc = self._get_cloud('ubuntu')
427 cc.distro = mock.MagicMock()
428- cc_lxd.handle('cc_lxd', {'lxd': {}}, cc, LOG, [])
429+ cc_lxd.handle('cc_lxd', {'lxd': {}}, cc, self.logger, [])
430 self.assertFalse(cc.distro.install_packages.called)
431 self.assertFalse(mock_util.subp.called)
432
433@@ -72,7 +72,7 @@ class TestLxd(t_help.TestCase):
434 def test_no_lxd_does_nothing(self, mock_util):
435 cc = self._get_cloud('ubuntu')
436 cc.distro = mock.MagicMock()
437- cc_lxd.handle('cc_lxd', {'package_update': True}, cc, LOG, [])
438+ cc_lxd.handle('cc_lxd', {'package_update': True}, cc, self.logger, [])
439 self.assertFalse(cc.distro.install_packages.called)
440 self.assertFalse(mock_util.subp.called)
441
442diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py
443index 4f29124..3abe578 100644
444--- a/tests/unittests/test_handler/test_handler_ntp.py
445+++ b/tests/unittests/test_handler/test_handler_ntp.py
446@@ -293,23 +293,24 @@ class TestNtp(FilesystemMockingTestCase):
447
448 def test_ntp_handler_schema_validation_allows_empty_ntp_config(self):
449 """Ntp schema validation allows for an empty ntp: configuration."""
450- invalid_config = {'ntp': {}}
451+ valid_empty_configs = [{'ntp': {}}, {'ntp': None}]
452 distro = 'ubuntu'
453 cc = self._get_cloud(distro)
454 ntp_conf = os.path.join(self.new_root, 'ntp.conf')
455 with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream:
456 stream.write(NTP_TEMPLATE)
457- with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf):
458- cc_ntp.handle('cc_ntp', invalid_config, cc, None, [])
459+ for valid_empty_config in valid_empty_configs:
460+ with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf):
461+ cc_ntp.handle('cc_ntp', valid_empty_config, cc, None, [])
462+ with open(ntp_conf) as stream:
463+ content = stream.read()
464+ default_pools = [
465+ "{0}.{1}.pool.ntp.org".format(x, distro)
466+ for x in range(0, cc_ntp.NR_POOL_SERVERS)]
467+ self.assertEqual(
468+ "servers []\npools {0}\n".format(default_pools),
469+ content)
470 self.assertNotIn('Invalid config:', self.logs.getvalue())
471- with open(ntp_conf) as stream:
472- content = stream.read()
473- default_pools = [
474- "{0}.{1}.pool.ntp.org".format(x, distro)
475- for x in range(0, cc_ntp.NR_POOL_SERVERS)]
476- self.assertEqual(
477- "servers []\npools {0}\n".format(default_pools),
478- content)
479
480 @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
481 def test_ntp_handler_schema_validation_warns_non_string_item_type(self):
482diff --git a/tests/unittests/test_handler/test_handler_resizefs.py b/tests/unittests/test_handler/test_handler_resizefs.py
483index 3e5d436..29d5574 100644
484--- a/tests/unittests/test_handler/test_handler_resizefs.py
485+++ b/tests/unittests/test_handler/test_handler_resizefs.py
486@@ -1,9 +1,9 @@
487 # This file is part of cloud-init. See LICENSE file for license information.
488
489 from cloudinit.config.cc_resizefs import (
490- can_skip_resize, handle, is_device_path_writable_block,
491- rootdev_from_cmdline)
492+ can_skip_resize, handle, maybe_get_writable_device_path)
493
494+from collections import namedtuple
495 import logging
496 import textwrap
497
498@@ -138,47 +138,48 @@ class TestRootDevFromCmdline(CiTestCase):
499 invalid_cases = [
500 'BOOT_IMAGE=/adsf asdfa werasef root adf', 'BOOT_IMAGE=/adsf', '']
501 for case in invalid_cases:
502- self.assertIsNone(rootdev_from_cmdline(case))
503+ self.assertIsNone(util.rootdev_from_cmdline(case))
504
505 def test_rootdev_from_cmdline_with_root_startswith_dev(self):
506 """Return the cmdline root when the path starts with /dev."""
507 self.assertEqual(
508- '/dev/this', rootdev_from_cmdline('asdf root=/dev/this'))
509+ '/dev/this', util.rootdev_from_cmdline('asdf root=/dev/this'))
510
511 def test_rootdev_from_cmdline_with_root_without_dev_prefix(self):
512 """Add /dev prefix to cmdline root when the path lacks the prefix."""
513- self.assertEqual('/dev/this', rootdev_from_cmdline('asdf root=this'))
514+ self.assertEqual(
515+ '/dev/this', util.rootdev_from_cmdline('asdf root=this'))
516
517 def test_rootdev_from_cmdline_with_root_with_label(self):
518 """When cmdline root contains a LABEL, our root is disk/by-label."""
519 self.assertEqual(
520 '/dev/disk/by-label/unique',
521- rootdev_from_cmdline('asdf root=LABEL=unique'))
522+ util.rootdev_from_cmdline('asdf root=LABEL=unique'))
523
524 def test_rootdev_from_cmdline_with_root_with_uuid(self):
525 """When cmdline root contains a UUID, our root is disk/by-uuid."""
526 self.assertEqual(
527 '/dev/disk/by-uuid/adsfdsaf-adsf',
528- rootdev_from_cmdline('asdf root=UUID=adsfdsaf-adsf'))
529+ util.rootdev_from_cmdline('asdf root=UUID=adsfdsaf-adsf'))
530
531
532-class TestIsDevicePathWritableBlock(CiTestCase):
533+class TestMaybeGetDevicePathAsWritableBlock(CiTestCase):
534
535 with_logs = True
536
537- def test_is_device_path_writable_block_false_on_overlayroot(self):
538+ def test_maybe_get_writable_device_path_none_on_overlayroot(self):
539 """When devpath is overlayroot (on MAAS), is_dev_writable is False."""
540 info = 'does not matter'
541- is_writable = wrap_and_call(
542+ devpath = wrap_and_call(
543 'cloudinit.config.cc_resizefs.util',
544 {'is_container': {'return_value': False}},
545- is_device_path_writable_block, 'overlayroot', info, LOG)
546- self.assertFalse(is_writable)
547+ maybe_get_writable_device_path, 'overlayroot', info, LOG)
548+ self.assertIsNone(devpath)
549 self.assertIn(
550 "Not attempting to resize devpath 'overlayroot'",
551 self.logs.getvalue())
552
553- def test_is_device_path_writable_block_warns_missing_cmdline_root(self):
554+ def test_maybe_get_writable_device_path_warns_missing_cmdline_root(self):
555 """When root does not exist isn't in the cmdline, log warning."""
556 info = 'does not matter'
557
558@@ -190,43 +191,43 @@ class TestIsDevicePathWritableBlock(CiTestCase):
559 exists_mock_path = 'cloudinit.config.cc_resizefs.os.path.exists'
560 with mock.patch(exists_mock_path) as m_exists:
561 m_exists.return_value = False
562- is_writable = wrap_and_call(
563+ devpath = wrap_and_call(
564 'cloudinit.config.cc_resizefs.util',
565 {'is_container': {'return_value': False},
566 'get_mount_info': {'side_effect': fake_mount_info},
567 'get_cmdline': {'return_value': 'BOOT_IMAGE=/vmlinuz.efi'}},
568- is_device_path_writable_block, '/dev/root', info, LOG)
569- self.assertFalse(is_writable)
570+ maybe_get_writable_device_path, '/dev/root', info, LOG)
571+ self.assertIsNone(devpath)
572 logs = self.logs.getvalue()
573 self.assertIn("WARNING: Unable to find device '/dev/root'", logs)
574
575- def test_is_device_path_writable_block_does_not_exist(self):
576+ def test_maybe_get_writable_device_path_does_not_exist(self):
577 """When devpath does not exist, a warning is logged."""
578 info = 'dev=/I/dont/exist mnt_point=/ path=/dev/none'
579- is_writable = wrap_and_call(
580+ devpath = wrap_and_call(
581 'cloudinit.config.cc_resizefs.util',
582 {'is_container': {'return_value': False}},
583- is_device_path_writable_block, '/I/dont/exist', info, LOG)
584- self.assertFalse(is_writable)
585+ maybe_get_writable_device_path, '/I/dont/exist', info, LOG)
586+ self.assertIsNone(devpath)
587 self.assertIn(
588 "WARNING: Device '/I/dont/exist' did not exist."
589 ' cannot resize: %s' % info,
590 self.logs.getvalue())
591
592- def test_is_device_path_writable_block_does_not_exist_in_container(self):
593+ def test_maybe_get_writable_device_path_does_not_exist_in_container(self):
594 """When devpath does not exist in a container, log a debug message."""
595 info = 'dev=/I/dont/exist mnt_point=/ path=/dev/none'
596- is_writable = wrap_and_call(
597+ devpath = wrap_and_call(
598 'cloudinit.config.cc_resizefs.util',
599 {'is_container': {'return_value': True}},
600- is_device_path_writable_block, '/I/dont/exist', info, LOG)
601- self.assertFalse(is_writable)
602+ maybe_get_writable_device_path, '/I/dont/exist', info, LOG)
603+ self.assertIsNone(devpath)
604 self.assertIn(
605 "DEBUG: Device '/I/dont/exist' did not exist in container."
606 ' cannot resize: %s' % info,
607 self.logs.getvalue())
608
609- def test_is_device_path_writable_block_raises_oserror(self):
610+ def test_maybe_get_writable_device_path_raises_oserror(self):
611 """When unexpected OSError is raises by os.stat it is reraised."""
612 info = 'dev=/I/dont/exist mnt_point=/ path=/dev/none'
613 with self.assertRaises(OSError) as context_manager:
614@@ -234,41 +235,63 @@ class TestIsDevicePathWritableBlock(CiTestCase):
615 'cloudinit.config.cc_resizefs',
616 {'util.is_container': {'return_value': True},
617 'os.stat': {'side_effect': OSError('Something unexpected')}},
618- is_device_path_writable_block, '/I/dont/exist', info, LOG)
619+ maybe_get_writable_device_path, '/I/dont/exist', info, LOG)
620 self.assertEqual(
621 'Something unexpected', str(context_manager.exception))
622
623- def test_is_device_path_writable_block_non_block(self):
624+ def test_maybe_get_writable_device_path_non_block(self):
625 """When device is not a block device, emit warning return False."""
626 fake_devpath = self.tmp_path('dev/readwrite')
627 util.write_file(fake_devpath, '', mode=0o600) # read-write
628 info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath)
629
630- is_writable = wrap_and_call(
631+ devpath = wrap_and_call(
632 'cloudinit.config.cc_resizefs.util',
633 {'is_container': {'return_value': False}},
634- is_device_path_writable_block, fake_devpath, info, LOG)
635- self.assertFalse(is_writable)
636+ maybe_get_writable_device_path, fake_devpath, info, LOG)
637+ self.assertIsNone(devpath)
638 self.assertIn(
639 "WARNING: device '{0}' not a block device. cannot resize".format(
640 fake_devpath),
641 self.logs.getvalue())
642
643- def test_is_device_path_writable_block_non_block_on_container(self):
644+ def test_maybe_get_writable_device_path_non_block_on_container(self):
645 """When device is non-block device in container, emit debug log."""
646 fake_devpath = self.tmp_path('dev/readwrite')
647 util.write_file(fake_devpath, '', mode=0o600) # read-write
648 info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath)
649
650- is_writable = wrap_and_call(
651+ devpath = wrap_and_call(
652 'cloudinit.config.cc_resizefs.util',
653 {'is_container': {'return_value': True}},
654- is_device_path_writable_block, fake_devpath, info, LOG)
655- self.assertFalse(is_writable)
656+ maybe_get_writable_device_path, fake_devpath, info, LOG)
657+ self.assertIsNone(devpath)
658 self.assertIn(
659 "DEBUG: device '{0}' not a block device in container."
660 ' cannot resize'.format(fake_devpath),
661 self.logs.getvalue())
662
663+ def test_maybe_get_writable_device_path_returns_cmdline_root(self):
664+ """When root device is UUID in kernel commandline, update devpath."""
665+ # XXX Long-term we want to use FilesystemMocking test to avoid
666+ # touching os.stat.
667+ FakeStat = namedtuple(
668+ 'FakeStat', ['st_mode', 'st_size', 'st_mtime']) # minimal def.
669+ info = 'dev=/dev/root mnt_point=/ path=/does/not/matter'
670+ devpath = wrap_and_call(
671+ 'cloudinit.config.cc_resizefs',
672+ {'util.get_cmdline': {'return_value': 'asdf root=UUID=my-uuid'},
673+ 'util.is_container': False,
674+ 'os.path.exists': False, # /dev/root doesn't exist
675+ 'os.stat': {
676+ 'return_value': FakeStat(25008, 0, 1)} # char block device
677+ },
678+ maybe_get_writable_device_path, '/dev/root', info, LOG)
679+ self.assertEqual('/dev/disk/by-uuid/my-uuid', devpath)
680+ self.assertIn(
681+ "DEBUG: Converted /dev/root to '/dev/disk/by-uuid/my-uuid'"
682+ " per kernel cmdline",
683+ self.logs.getvalue())
684+
685
686 # vi: ts=4 expandtab
687diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py
688index b8fc893..648573f 100644
689--- a/tests/unittests/test_handler/test_schema.py
690+++ b/tests/unittests/test_handler/test_schema.py
691@@ -4,11 +4,12 @@ from cloudinit.config.schema import (
692 CLOUD_CONFIG_HEADER, SchemaValidationError, annotated_cloudconfig_file,
693 get_schema_doc, get_schema, validate_cloudconfig_file,
694 validate_cloudconfig_schema, main)
695-from cloudinit.util import write_file
696+from cloudinit.util import subp, write_file
697
698 from cloudinit.tests.helpers import CiTestCase, mock, skipIf
699
700 from copy import copy
701+import os
702 from six import StringIO
703 from textwrap import dedent
704 from yaml import safe_load
705@@ -364,4 +365,38 @@ class MainTest(CiTestCase):
706 self.assertIn(
707 'Valid cloud-config file {0}'.format(myyaml), m_stdout.getvalue())
708
709+
710+class CloudTestsIntegrationTest(CiTestCase):
711+ """Validate all cloud-config yaml schema provided in integration tests.
712+
713+ It is less expensive to have unittests validate schema of all cloud-config
714+ yaml provided to integration tests, than to run an integration test which
715+ raises Warnings or errors on invalid cloud-config schema.
716+ """
717+
718+ @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency")
719+ def test_all_integration_test_cloud_config_schema(self):
720+ """Validate schema of cloud_tests yaml files looking for warnings."""
721+ schema = get_schema()
722+ testsdir = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
723+ integration_testdir = os.path.sep.join(
724+ [testsdir, 'cloud_tests', 'testcases'])
725+ errors = []
726+ out, _ = subp(['find', integration_testdir, '-name', '*yaml'])
727+ for filename in out.splitlines():
728+ test_cfg = safe_load(open(filename))
729+ cloud_config = test_cfg.get('cloud_config')
730+ if cloud_config:
731+ cloud_config = safe_load(
732+ cloud_config.replace("#cloud-config\n", ""))
733+ try:
734+ validate_cloudconfig_schema(
735+ cloud_config, schema, strict=True)
736+ except SchemaValidationError as e:
737+ errors.append(
738+ '{0}: {1}'.format(
739+ filename, e))
740+ if errors:
741+ raise AssertionError(', '.join(errors))
742+
743 # vi: ts=4 expandtab syntax=python
744diff --git a/tools/read-dependencies b/tools/read-dependencies
745index 2a64868..421f470 100755
746--- a/tools/read-dependencies
747+++ b/tools/read-dependencies
748@@ -30,9 +30,35 @@ DISTRO_PKG_TYPE_MAP = {
749 'suse': 'suse'
750 }
751
752-DISTRO_INSTALL_PKG_CMD = {
753+MAYBE_RELIABLE_YUM_INSTALL = [
754+ 'sh', '-c',
755+ """
756+ error() { echo "$@" 1>&2; }
757+ n=0; max=10;
758+ bcmd="yum install --downloadonly --assumeyes --setopt=keepcache=1"
759+ while n=$(($n+1)); do
760+ error ":: running $bcmd $* [$n/$max]"
761+ $bcmd "$@"
762+ r=$?
763+ [ $r -eq 0 ] && break
764+ [ $n -ge $max ] && { error "gave up on $bcmd"; exit $r; }
765+ nap=$(($n*5))
766+ error ":: failed [$r] ($n/$max). sleeping $nap."
767+ sleep $nap
768+ done
769+ error ":: running yum install --cacheonly --assumeyes $*"
770+ yum install --cacheonly --assumeyes "$@"
771+ """,
772+ 'reliable-yum-install']
773+
774+DRY_DISTRO_INSTALL_PKG_CMD = {
775 'centos': ['yum', 'install', '--assumeyes'],
776 'redhat': ['yum', 'install', '--assumeyes'],
777+}
778+
779+DISTRO_INSTALL_PKG_CMD = {
780+ 'centos': MAYBE_RELIABLE_YUM_INSTALL,
781+ 'redhat': MAYBE_RELIABLE_YUM_INSTALL,
782 'debian': ['apt', 'install', '-y'],
783 'ubuntu': ['apt', 'install', '-y'],
784 'opensuse': ['zypper', 'install'],
785@@ -80,8 +106,8 @@ def get_parser():
786 help='Additionally install continuous integration system packages '
787 'required for build and test automation.')
788 parser.add_argument(
789- '-v', '--python-version', type=str, dest='python_version', default=None,
790- choices=["2", "3"],
791+ '-v', '--python-version', type=str, dest='python_version',
792+ default=None, choices=["2", "3"],
793 help='Override the version of python we want to generate system '
794 'package dependencies for. Defaults to the version of python '
795 'this script is called with')
796@@ -219,10 +245,15 @@ def pkg_install(pkg_list, distro, test_distro=False, dry_run=False):
797 '(dryrun)' if dry_run else '', ' '.join(pkg_list)))
798 install_cmd = []
799 if dry_run:
800- install_cmd.append('echo')
801+ install_cmd.append('echo')
802 if os.geteuid() != 0:
803 install_cmd.append('sudo')
804- install_cmd.extend(DISTRO_INSTALL_PKG_CMD[distro])
805+
806+ cmd = DISTRO_INSTALL_PKG_CMD[distro]
807+ if dry_run and distro in DRY_DISTRO_INSTALL_PKG_CMD:
808+ cmd = DRY_DISTRO_INSTALL_PKG_CMD[distro]
809+ install_cmd.extend(cmd)
810+
811 if distro in ['centos', 'redhat']:
812 # CentOS and Redhat need epel-release to access oauthlib and jsonschema
813 subprocess.check_call(install_cmd + ['epel-release'])
814diff --git a/tools/run-centos b/tools/run-centos
815index d44d514..d58ef3e 100755
816--- a/tools/run-centos
817+++ b/tools/run-centos
818@@ -123,7 +123,22 @@ prep() {
819 return 0
820 fi
821 error "Installing prep packages: ${needed}"
822- yum install --assumeyes ${needed}
823+ set -- $needed
824+ local n max r
825+ n=0; max=10;
826+ bcmd="yum install --downloadonly --assumeyes --setopt=keepcache=1"
827+ while n=$(($n+1)); do
828+ error ":: running $bcmd $* [$n/$max]"
829+ $bcmd "$@"
830+ r=$?
831+ [ $r -eq 0 ] && break
832+ [ $n -ge $max ] && { error "gave up on $bcmd"; exit $r; }
833+ nap=$(($n*5))
834+ error ":: failed [$r] ($n/$max). sleeping $nap."
835+ sleep $nap
836+ done
837+ error ":: running yum install --cacheonly --assumeyes $*"
838+ yum install --cacheonly --assumeyes "$@"
839 }
840
841 start_container() {
842@@ -153,6 +168,7 @@ start_container() {
843 if [ ! -z "${http_proxy-}" ]; then
844 debug 1 "configuring proxy ${http_proxy}"
845 inside "$name" sh -c "echo proxy=$http_proxy >> /etc/yum.conf"
846+ inside "$name" sed -i s/enabled=1/enabled=0/ /etc/yum/pluginconf.d/fastestmirror.conf
847 fi
848 }
849

Subscribers

People subscribed via source and target branches