Merge ~chad.smith/cloud-init:ubuntu/artful into cloud-init:ubuntu/artful
- Git
- lp:~chad.smith/cloud-init
- ubuntu/artful
- Merge into ubuntu/artful
Proposed by
Chad Smith
Status: | Merged |
---|---|
Merged at revision: | a9968540aa273bee10b0ca040133c01aa3459793 |
Proposed branch: | ~chad.smith/cloud-init:ubuntu/artful |
Merge into: | cloud-init:ubuntu/artful |
Diff against target: |
848 lines (+260/-107) 21 files modified
cloudinit/config/cc_lxd.py (+1/-1) cloudinit/config/cc_ntp.py (+3/-1) cloudinit/config/cc_resizefs.py (+13/-30) cloudinit/config/cc_users_groups.py (+2/-1) cloudinit/config/schema.py (+1/-1) debian/changelog (+16/-0) doc/examples/cloud-config-user-groups.txt (+3/-3) tests/cloud_tests/testcases/__init__.py (+7/-0) tests/cloud_tests/testcases/base.py (+8/-4) tests/cloud_tests/testcases/examples/including_user_groups.py (+6/-0) tests/cloud_tests/testcases/examples/including_user_groups.yaml (+5/-2) tests/cloud_tests/testcases/main/command_output_simple.py (+16/-0) tests/cloud_tests/testcases/modules/ntp.yaml (+2/-2) tests/cloud_tests/testcases/modules/user_groups.py (+6/-0) tests/cloud_tests/testcases/modules/user_groups.yaml (+5/-2) tests/unittests/test_handler/test_handler_lxd.py (+8/-8) tests/unittests/test_handler/test_handler_ntp.py (+12/-11) tests/unittests/test_handler/test_handler_resizefs.py (+57/-34) tests/unittests/test_handler/test_schema.py (+36/-1) tools/read-dependencies (+36/-5) tools/run-centos (+17/-1) |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Server Team CI bot | continuous-integration | Approve | |
Scott Moser | Pending | ||
Review via email: mp+332673@code.launchpad.net |
Commit message
Description of the change
Upstream snapshot into Artful for SRU
To post a comment you must log in.
Revision history for this message
Server Team CI bot (server-team-bot) wrote : | # |
review:
Approve
(continuous-integration)
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py |
2 | index e6262f8..09374d2 100644 |
3 | --- a/cloudinit/config/cc_lxd.py |
4 | +++ b/cloudinit/config/cc_lxd.py |
5 | @@ -72,7 +72,7 @@ def handle(name, cfg, cloud, log, args): |
6 | type(init_cfg)) |
7 | init_cfg = {} |
8 | |
9 | - bridge_cfg = lxd_cfg.get('bridge') |
10 | + bridge_cfg = lxd_cfg.get('bridge', {}) |
11 | if not isinstance(bridge_cfg, dict): |
12 | log.warn("lxd/bridge config must be a dictionary. found a '%s'", |
13 | type(bridge_cfg)) |
14 | diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py |
15 | index 15ae1ec..d43d060 100644 |
16 | --- a/cloudinit/config/cc_ntp.py |
17 | +++ b/cloudinit/config/cc_ntp.py |
18 | @@ -100,7 +100,9 @@ def handle(name, cfg, cloud, log, _args): |
19 | LOG.debug( |
20 | "Skipping module named %s, not present or disabled by cfg", name) |
21 | return |
22 | - ntp_cfg = cfg.get('ntp', {}) |
23 | + ntp_cfg = cfg['ntp'] |
24 | + if ntp_cfg is None: |
25 | + ntp_cfg = {} # Allow empty config which will install the package |
26 | |
27 | # TODO drop this when validate_cloudconfig_schema is strict=True |
28 | if not isinstance(ntp_cfg, (dict)): |
29 | diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py |
30 | index f774baa..0d282e6 100644 |
31 | --- a/cloudinit/config/cc_resizefs.py |
32 | +++ b/cloudinit/config/cc_resizefs.py |
33 | @@ -145,25 +145,6 @@ RESIZE_FS_PRECHECK_CMDS = { |
34 | } |
35 | |
36 | |
37 | -def rootdev_from_cmdline(cmdline): |
38 | - found = None |
39 | - for tok in cmdline.split(): |
40 | - if tok.startswith("root="): |
41 | - found = tok[5:] |
42 | - break |
43 | - if found is None: |
44 | - return None |
45 | - |
46 | - if found.startswith("/dev/"): |
47 | - return found |
48 | - if found.startswith("LABEL="): |
49 | - return "/dev/disk/by-label/" + found[len("LABEL="):] |
50 | - if found.startswith("UUID="): |
51 | - return "/dev/disk/by-uuid/" + found[len("UUID="):] |
52 | - |
53 | - return "/dev/" + found |
54 | - |
55 | - |
56 | def can_skip_resize(fs_type, resize_what, devpth): |
57 | fstype_lc = fs_type.lower() |
58 | for i, func in RESIZE_FS_PRECHECK_CMDS.items(): |
59 | @@ -172,14 +153,15 @@ def can_skip_resize(fs_type, resize_what, devpth): |
60 | return False |
61 | |
62 | |
63 | -def is_device_path_writable_block(devpath, info, log): |
64 | - """Return True if devpath is a writable block device. |
65 | +def maybe_get_writable_device_path(devpath, info, log): |
66 | + """Return updated devpath if the devpath is a writable block device. |
67 | |
68 | - @param devpath: Path to the root device we want to resize. |
69 | + @param devpath: Requested path to the root device we want to resize. |
70 | @param info: String representing information about the requested device. |
71 | @param log: Logger to which logs will be added upon error. |
72 | |
73 | - @returns Boolean True if block device is writable |
74 | + @returns devpath or updated devpath per kernel commandline if the device |
75 | + path is a writable block device, returns None otherwise. |
76 | """ |
77 | container = util.is_container() |
78 | |
79 | @@ -189,12 +171,12 @@ def is_device_path_writable_block(devpath, info, log): |
80 | devpath = util.rootdev_from_cmdline(util.get_cmdline()) |
81 | if devpath is None: |
82 | log.warn("Unable to find device '/dev/root'") |
83 | - return False |
84 | + return None |
85 | log.debug("Converted /dev/root to '%s' per kernel cmdline", devpath) |
86 | |
87 | if devpath == 'overlayroot': |
88 | log.debug("Not attempting to resize devpath '%s': %s", devpath, info) |
89 | - return False |
90 | + return None |
91 | |
92 | try: |
93 | statret = os.stat(devpath) |
94 | @@ -207,7 +189,7 @@ def is_device_path_writable_block(devpath, info, log): |
95 | devpath, info) |
96 | else: |
97 | raise exc |
98 | - return False |
99 | + return None |
100 | |
101 | if not stat.S_ISBLK(statret.st_mode) and not stat.S_ISCHR(statret.st_mode): |
102 | if container: |
103 | @@ -216,8 +198,8 @@ def is_device_path_writable_block(devpath, info, log): |
104 | else: |
105 | log.warn("device '%s' not a block device. cannot resize: %s" % |
106 | (devpath, info)) |
107 | - return False |
108 | - return True |
109 | + return None |
110 | + return devpath # The writable block devpath |
111 | |
112 | |
113 | def handle(name, cfg, _cloud, log, args): |
114 | @@ -242,8 +224,9 @@ def handle(name, cfg, _cloud, log, args): |
115 | info = "dev=%s mnt_point=%s path=%s" % (devpth, mount_point, resize_what) |
116 | log.debug("resize_info: %s" % info) |
117 | |
118 | - if not is_device_path_writable_block(devpth, info, log): |
119 | - return |
120 | + devpth = maybe_get_writable_device_path(devpth, info, log) |
121 | + if not devpth: |
122 | + return # devpath was not a writable block device |
123 | |
124 | resizer = None |
125 | if can_skip_resize(fs_type, resize_what, devpth): |
126 | diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py |
127 | index b80d1d3..f363000 100644 |
128 | --- a/cloudinit/config/cc_users_groups.py |
129 | +++ b/cloudinit/config/cc_users_groups.py |
130 | @@ -15,7 +15,8 @@ options, see the ``Including users and groups`` config example. |
131 | Groups to add to the system can be specified as a list under the ``groups`` |
132 | key. Each entry in the list should either contain a the group name as a string, |
133 | or a dictionary with the group name as the key and a list of users who should |
134 | -be members of the group as the value. |
135 | +be members of the group as the value. **Note**: Groups are added before users, |
136 | +so any users in a group list must already exist on the system. |
137 | |
138 | The ``users`` config key takes a list of users to configure. The first entry in |
139 | this list is used as the default user for the system. To preserve the standard |
140 | diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py |
141 | index bb291ff..ca7d0d5 100644 |
142 | --- a/cloudinit/config/schema.py |
143 | +++ b/cloudinit/config/schema.py |
144 | @@ -74,7 +74,7 @@ def validate_cloudconfig_schema(config, schema, strict=False): |
145 | try: |
146 | from jsonschema import Draft4Validator, FormatChecker |
147 | except ImportError: |
148 | - logging.warning( |
149 | + logging.debug( |
150 | 'Ignoring schema validation. python-jsonschema is not present') |
151 | return |
152 | validator = Draft4Validator(schema, format_checker=FormatChecker()) |
153 | diff --git a/debian/changelog b/debian/changelog |
154 | index 26d1d45..5f0c1ce 100644 |
155 | --- a/debian/changelog |
156 | +++ b/debian/changelog |
157 | @@ -1,3 +1,19 @@ |
158 | +cloud-init (17.1-25-g17a15f9e-0ubuntu1~17.10.1) artful-proposed; urgency=medium |
159 | + |
160 | + * New upstream snapshot. |
161 | + - resizefs: Fix regression when system booted with root=PARTUUID= |
162 | + (LP: #1725067) |
163 | + - tools: make yum package installation more reliable |
164 | + - citest: fix remaining warnings raised by integration tests. |
165 | + - citest: show the class actual class name in results. |
166 | + - ntp: fix config module schema to allow empty ntp config |
167 | + (LP: #1724951) |
168 | + - tools: disable fastestmirror if using proxy [Joshua Powers] |
169 | + - schema: Log debug instead of warning when jsonschema is not available. |
170 | + (LP: #1724354) |
171 | + |
172 | + -- Chad Smith <chad.smith@canonical.com> Mon, 23 Oct 2017 15:07:35 -0600 |
173 | + |
174 | cloud-init (17.1-18-gd4f70470-0ubuntu1) artful; urgency=medium |
175 | |
176 | * New upstream snapshot. |
177 | diff --git a/doc/examples/cloud-config-user-groups.txt b/doc/examples/cloud-config-user-groups.txt |
178 | index 9c5202f..0554d1f 100644 |
179 | --- a/doc/examples/cloud-config-user-groups.txt |
180 | +++ b/doc/examples/cloud-config-user-groups.txt |
181 | @@ -1,8 +1,8 @@ |
182 | # Add groups to the system |
183 | -# The following example adds the ubuntu group with members foo and bar and |
184 | -# the group cloud-users. |
185 | +# The following example adds the ubuntu group with members 'root' and 'sys' |
186 | +# and the empty group cloud-users. |
187 | groups: |
188 | - - ubuntu: [foo,bar] |
189 | + - ubuntu: [root,sys] |
190 | - cloud-users |
191 | |
192 | # Add users to the system. Users are added after groups are added. |
193 | diff --git a/tests/cloud_tests/testcases/__init__.py b/tests/cloud_tests/testcases/__init__.py |
194 | index 47217ce..a29a092 100644 |
195 | --- a/tests/cloud_tests/testcases/__init__.py |
196 | +++ b/tests/cloud_tests/testcases/__init__.py |
197 | @@ -5,6 +5,7 @@ |
198 | import importlib |
199 | import inspect |
200 | import unittest |
201 | +from unittest.util import strclass |
202 | |
203 | from tests.cloud_tests import config |
204 | from tests.cloud_tests.testcases.base import CloudTestCase as base_test |
205 | @@ -37,6 +38,12 @@ def get_suite(test_name, data, conf): |
206 | |
207 | class tmp(test_class): |
208 | |
209 | + _realclass = test_class |
210 | + |
211 | + def __str__(self): |
212 | + return "%s (%s)" % (self._testMethodName, |
213 | + strclass(self._realclass)) |
214 | + |
215 | @classmethod |
216 | def setUpClass(cls): |
217 | cls.data = data |
218 | diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py |
219 | index bb545ab..1706f59 100644 |
220 | --- a/tests/cloud_tests/testcases/base.py |
221 | +++ b/tests/cloud_tests/testcases/base.py |
222 | @@ -16,10 +16,6 @@ class CloudTestCase(unittest.TestCase): |
223 | conf = None |
224 | _cloud_config = None |
225 | |
226 | - def shortDescription(self): |
227 | - """Prevent nose from using docstrings.""" |
228 | - return None |
229 | - |
230 | @property |
231 | def cloud_config(self): |
232 | """Get the cloud-config used by the test.""" |
233 | @@ -72,6 +68,14 @@ class CloudTestCase(unittest.TestCase): |
234 | result = self.get_status_data(self.get_data_file('result.json')) |
235 | self.assertEqual(len(result['errors']), 0) |
236 | |
237 | + def test_no_warnings_in_log(self): |
238 | + """Warnings should not be found in the log.""" |
239 | + self.assertEqual( |
240 | + [], |
241 | + [l for l in self.get_data_file('cloud-init.log').splitlines() |
242 | + if 'WARN' in l], |
243 | + msg="'WARN' found inside cloud-init.log") |
244 | + |
245 | |
246 | class PasswordListTest(CloudTestCase): |
247 | """Base password test case class.""" |
248 | diff --git a/tests/cloud_tests/testcases/examples/including_user_groups.py b/tests/cloud_tests/testcases/examples/including_user_groups.py |
249 | index 67af527..93b7a82 100644 |
250 | --- a/tests/cloud_tests/testcases/examples/including_user_groups.py |
251 | +++ b/tests/cloud_tests/testcases/examples/including_user_groups.py |
252 | @@ -40,4 +40,10 @@ class TestUserGroups(base.CloudTestCase): |
253 | out = self.get_data_file('user_cloudy') |
254 | self.assertRegex(out, r'cloudy:x:[0-9]{3,4}:') |
255 | |
256 | + def test_user_root_in_secret(self): |
257 | + """Test root user is in 'secret' group.""" |
258 | + user, _, groups = self.get_data_file('root_groups').partition(":") |
259 | + self.assertIn("secret", groups.split(), |
260 | + msg="User root is not in group 'secret'") |
261 | + |
262 | # vi: ts=4 expandtab |
263 | diff --git a/tests/cloud_tests/testcases/examples/including_user_groups.yaml b/tests/cloud_tests/testcases/examples/including_user_groups.yaml |
264 | index 0aa7ad2..469d03c 100644 |
265 | --- a/tests/cloud_tests/testcases/examples/including_user_groups.yaml |
266 | +++ b/tests/cloud_tests/testcases/examples/including_user_groups.yaml |
267 | @@ -8,7 +8,7 @@ cloud_config: | |
268 | #cloud-config |
269 | # Add groups to the system |
270 | groups: |
271 | - - secret: [foobar,barfoo] |
272 | + - secret: [root] |
273 | - cloud-users |
274 | |
275 | # Add users to the system. Users are added after groups are added. |
276 | @@ -24,7 +24,7 @@ cloud_config: | |
277 | - name: barfoo |
278 | gecos: Bar B. Foo |
279 | sudo: ALL=(ALL) NOPASSWD:ALL |
280 | - groups: cloud-users |
281 | + groups: [cloud-users, secret] |
282 | lock_passwd: true |
283 | - name: cloudy |
284 | gecos: Magic Cloud App Daemon User |
285 | @@ -49,5 +49,8 @@ collect_scripts: |
286 | user_cloudy: | |
287 | #!/bin/bash |
288 | getent passwd cloudy |
289 | + root_groups: | |
290 | + #!/bin/bash |
291 | + groups root |
292 | |
293 | # vi: ts=4 expandtab |
294 | diff --git a/tests/cloud_tests/testcases/main/command_output_simple.py b/tests/cloud_tests/testcases/main/command_output_simple.py |
295 | index fe4c767..857881c 100644 |
296 | --- a/tests/cloud_tests/testcases/main/command_output_simple.py |
297 | +++ b/tests/cloud_tests/testcases/main/command_output_simple.py |
298 | @@ -15,4 +15,20 @@ class TestCommandOutputSimple(base.CloudTestCase): |
299 | data.splitlines()[-1].strip()) |
300 | # TODO: need to test that all stages redirected here |
301 | |
302 | + def test_no_warnings_in_log(self): |
303 | + """Warnings should not be found in the log. |
304 | + |
305 | + This class redirected stderr and stdout, so it expects to find |
306 | + a warning in cloud-init.log to that effect.""" |
307 | + redirect_msg = 'Stdout, stderr changing to' |
308 | + warnings = [ |
309 | + l for l in self.get_data_file('cloud-init.log').splitlines() |
310 | + if 'WARN' in l] |
311 | + self.assertEqual( |
312 | + [], [w for w in warnings if redirect_msg not in w], |
313 | + msg="'WARN' found inside cloud-init.log") |
314 | + self.assertEqual( |
315 | + 1, len(warnings), |
316 | + msg="Did not find %s in cloud-init.log" % redirect_msg) |
317 | + |
318 | # vi: ts=4 expandtab |
319 | diff --git a/tests/cloud_tests/testcases/modules/ntp.yaml b/tests/cloud_tests/testcases/modules/ntp.yaml |
320 | index fbef431..2530d72 100644 |
321 | --- a/tests/cloud_tests/testcases/modules/ntp.yaml |
322 | +++ b/tests/cloud_tests/testcases/modules/ntp.yaml |
323 | @@ -4,8 +4,8 @@ |
324 | cloud_config: | |
325 | #cloud-config |
326 | ntp: |
327 | - pools: {} |
328 | - servers: {} |
329 | + pools: [] |
330 | + servers: [] |
331 | collect_scripts: |
332 | ntp_installed: | |
333 | #!/bin/bash |
334 | diff --git a/tests/cloud_tests/testcases/modules/user_groups.py b/tests/cloud_tests/testcases/modules/user_groups.py |
335 | index 67af527..93b7a82 100644 |
336 | --- a/tests/cloud_tests/testcases/modules/user_groups.py |
337 | +++ b/tests/cloud_tests/testcases/modules/user_groups.py |
338 | @@ -40,4 +40,10 @@ class TestUserGroups(base.CloudTestCase): |
339 | out = self.get_data_file('user_cloudy') |
340 | self.assertRegex(out, r'cloudy:x:[0-9]{3,4}:') |
341 | |
342 | + def test_user_root_in_secret(self): |
343 | + """Test root user is in 'secret' group.""" |
344 | + user, _, groups = self.get_data_file('root_groups').partition(":") |
345 | + self.assertIn("secret", groups.split(), |
346 | + msg="User root is not in group 'secret'") |
347 | + |
348 | # vi: ts=4 expandtab |
349 | diff --git a/tests/cloud_tests/testcases/modules/user_groups.yaml b/tests/cloud_tests/testcases/modules/user_groups.yaml |
350 | index 71cc9da..22b5d70 100644 |
351 | --- a/tests/cloud_tests/testcases/modules/user_groups.yaml |
352 | +++ b/tests/cloud_tests/testcases/modules/user_groups.yaml |
353 | @@ -7,7 +7,7 @@ cloud_config: | |
354 | #cloud-config |
355 | # Add groups to the system |
356 | groups: |
357 | - - secret: [foobar,barfoo] |
358 | + - secret: [root] |
359 | - cloud-users |
360 | |
361 | # Add users to the system. Users are added after groups are added. |
362 | @@ -23,7 +23,7 @@ cloud_config: | |
363 | - name: barfoo |
364 | gecos: Bar B. Foo |
365 | sudo: ALL=(ALL) NOPASSWD:ALL |
366 | - groups: cloud-users |
367 | + groups: [cloud-users, secret] |
368 | lock_passwd: true |
369 | - name: cloudy |
370 | gecos: Magic Cloud App Daemon User |
371 | @@ -48,5 +48,8 @@ collect_scripts: |
372 | user_cloudy: | |
373 | #!/bin/bash |
374 | getent passwd cloudy |
375 | + root_groups: | |
376 | + #!/bin/bash |
377 | + groups root |
378 | |
379 | # vi: ts=4 expandtab |
380 | diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py |
381 | index f132a77..e0d9ab6 100644 |
382 | --- a/tests/unittests/test_handler/test_handler_lxd.py |
383 | +++ b/tests/unittests/test_handler/test_handler_lxd.py |
384 | @@ -5,17 +5,16 @@ from cloudinit.sources import DataSourceNoCloud |
385 | from cloudinit import (distros, helpers, cloud) |
386 | from cloudinit.tests import helpers as t_help |
387 | |
388 | -import logging |
389 | - |
390 | try: |
391 | from unittest import mock |
392 | except ImportError: |
393 | import mock |
394 | |
395 | -LOG = logging.getLogger(__name__) |
396 | |
397 | +class TestLxd(t_help.CiTestCase): |
398 | + |
399 | + with_logs = True |
400 | |
401 | -class TestLxd(t_help.TestCase): |
402 | lxd_cfg = { |
403 | 'lxd': { |
404 | 'init': { |
405 | @@ -41,7 +40,7 @@ class TestLxd(t_help.TestCase): |
406 | def test_lxd_init(self, mock_util): |
407 | cc = self._get_cloud('ubuntu') |
408 | mock_util.which.return_value = True |
409 | - cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, LOG, []) |
410 | + cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, self.logger, []) |
411 | self.assertTrue(mock_util.which.called) |
412 | init_call = mock_util.subp.call_args_list[0][0][0] |
413 | self.assertEqual(init_call, |
414 | @@ -55,7 +54,8 @@ class TestLxd(t_help.TestCase): |
415 | cc = self._get_cloud('ubuntu') |
416 | cc.distro = mock.MagicMock() |
417 | mock_util.which.return_value = None |
418 | - cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, LOG, []) |
419 | + cc_lxd.handle('cc_lxd', self.lxd_cfg, cc, self.logger, []) |
420 | + self.assertNotIn('WARN', self.logs.getvalue()) |
421 | self.assertTrue(cc.distro.install_packages.called) |
422 | install_pkg = cc.distro.install_packages.call_args_list[0][0][0] |
423 | self.assertEqual(sorted(install_pkg), ['lxd', 'zfs']) |
424 | @@ -64,7 +64,7 @@ class TestLxd(t_help.TestCase): |
425 | def test_no_init_does_nothing(self, mock_util): |
426 | cc = self._get_cloud('ubuntu') |
427 | cc.distro = mock.MagicMock() |
428 | - cc_lxd.handle('cc_lxd', {'lxd': {}}, cc, LOG, []) |
429 | + cc_lxd.handle('cc_lxd', {'lxd': {}}, cc, self.logger, []) |
430 | self.assertFalse(cc.distro.install_packages.called) |
431 | self.assertFalse(mock_util.subp.called) |
432 | |
433 | @@ -72,7 +72,7 @@ class TestLxd(t_help.TestCase): |
434 | def test_no_lxd_does_nothing(self, mock_util): |
435 | cc = self._get_cloud('ubuntu') |
436 | cc.distro = mock.MagicMock() |
437 | - cc_lxd.handle('cc_lxd', {'package_update': True}, cc, LOG, []) |
438 | + cc_lxd.handle('cc_lxd', {'package_update': True}, cc, self.logger, []) |
439 | self.assertFalse(cc.distro.install_packages.called) |
440 | self.assertFalse(mock_util.subp.called) |
441 | |
442 | diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py |
443 | index 4f29124..3abe578 100644 |
444 | --- a/tests/unittests/test_handler/test_handler_ntp.py |
445 | +++ b/tests/unittests/test_handler/test_handler_ntp.py |
446 | @@ -293,23 +293,24 @@ class TestNtp(FilesystemMockingTestCase): |
447 | |
448 | def test_ntp_handler_schema_validation_allows_empty_ntp_config(self): |
449 | """Ntp schema validation allows for an empty ntp: configuration.""" |
450 | - invalid_config = {'ntp': {}} |
451 | + valid_empty_configs = [{'ntp': {}}, {'ntp': None}] |
452 | distro = 'ubuntu' |
453 | cc = self._get_cloud(distro) |
454 | ntp_conf = os.path.join(self.new_root, 'ntp.conf') |
455 | with open('{0}.tmpl'.format(ntp_conf), 'wb') as stream: |
456 | stream.write(NTP_TEMPLATE) |
457 | - with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf): |
458 | - cc_ntp.handle('cc_ntp', invalid_config, cc, None, []) |
459 | + for valid_empty_config in valid_empty_configs: |
460 | + with mock.patch('cloudinit.config.cc_ntp.NTP_CONF', ntp_conf): |
461 | + cc_ntp.handle('cc_ntp', valid_empty_config, cc, None, []) |
462 | + with open(ntp_conf) as stream: |
463 | + content = stream.read() |
464 | + default_pools = [ |
465 | + "{0}.{1}.pool.ntp.org".format(x, distro) |
466 | + for x in range(0, cc_ntp.NR_POOL_SERVERS)] |
467 | + self.assertEqual( |
468 | + "servers []\npools {0}\n".format(default_pools), |
469 | + content) |
470 | self.assertNotIn('Invalid config:', self.logs.getvalue()) |
471 | - with open(ntp_conf) as stream: |
472 | - content = stream.read() |
473 | - default_pools = [ |
474 | - "{0}.{1}.pool.ntp.org".format(x, distro) |
475 | - for x in range(0, cc_ntp.NR_POOL_SERVERS)] |
476 | - self.assertEqual( |
477 | - "servers []\npools {0}\n".format(default_pools), |
478 | - content) |
479 | |
480 | @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency") |
481 | def test_ntp_handler_schema_validation_warns_non_string_item_type(self): |
482 | diff --git a/tests/unittests/test_handler/test_handler_resizefs.py b/tests/unittests/test_handler/test_handler_resizefs.py |
483 | index 3e5d436..29d5574 100644 |
484 | --- a/tests/unittests/test_handler/test_handler_resizefs.py |
485 | +++ b/tests/unittests/test_handler/test_handler_resizefs.py |
486 | @@ -1,9 +1,9 @@ |
487 | # This file is part of cloud-init. See LICENSE file for license information. |
488 | |
489 | from cloudinit.config.cc_resizefs import ( |
490 | - can_skip_resize, handle, is_device_path_writable_block, |
491 | - rootdev_from_cmdline) |
492 | + can_skip_resize, handle, maybe_get_writable_device_path) |
493 | |
494 | +from collections import namedtuple |
495 | import logging |
496 | import textwrap |
497 | |
498 | @@ -138,47 +138,48 @@ class TestRootDevFromCmdline(CiTestCase): |
499 | invalid_cases = [ |
500 | 'BOOT_IMAGE=/adsf asdfa werasef root adf', 'BOOT_IMAGE=/adsf', ''] |
501 | for case in invalid_cases: |
502 | - self.assertIsNone(rootdev_from_cmdline(case)) |
503 | + self.assertIsNone(util.rootdev_from_cmdline(case)) |
504 | |
505 | def test_rootdev_from_cmdline_with_root_startswith_dev(self): |
506 | """Return the cmdline root when the path starts with /dev.""" |
507 | self.assertEqual( |
508 | - '/dev/this', rootdev_from_cmdline('asdf root=/dev/this')) |
509 | + '/dev/this', util.rootdev_from_cmdline('asdf root=/dev/this')) |
510 | |
511 | def test_rootdev_from_cmdline_with_root_without_dev_prefix(self): |
512 | """Add /dev prefix to cmdline root when the path lacks the prefix.""" |
513 | - self.assertEqual('/dev/this', rootdev_from_cmdline('asdf root=this')) |
514 | + self.assertEqual( |
515 | + '/dev/this', util.rootdev_from_cmdline('asdf root=this')) |
516 | |
517 | def test_rootdev_from_cmdline_with_root_with_label(self): |
518 | """When cmdline root contains a LABEL, our root is disk/by-label.""" |
519 | self.assertEqual( |
520 | '/dev/disk/by-label/unique', |
521 | - rootdev_from_cmdline('asdf root=LABEL=unique')) |
522 | + util.rootdev_from_cmdline('asdf root=LABEL=unique')) |
523 | |
524 | def test_rootdev_from_cmdline_with_root_with_uuid(self): |
525 | """When cmdline root contains a UUID, our root is disk/by-uuid.""" |
526 | self.assertEqual( |
527 | '/dev/disk/by-uuid/adsfdsaf-adsf', |
528 | - rootdev_from_cmdline('asdf root=UUID=adsfdsaf-adsf')) |
529 | + util.rootdev_from_cmdline('asdf root=UUID=adsfdsaf-adsf')) |
530 | |
531 | |
532 | -class TestIsDevicePathWritableBlock(CiTestCase): |
533 | +class TestMaybeGetDevicePathAsWritableBlock(CiTestCase): |
534 | |
535 | with_logs = True |
536 | |
537 | - def test_is_device_path_writable_block_false_on_overlayroot(self): |
538 | + def test_maybe_get_writable_device_path_none_on_overlayroot(self): |
539 | """When devpath is overlayroot (on MAAS), is_dev_writable is False.""" |
540 | info = 'does not matter' |
541 | - is_writable = wrap_and_call( |
542 | + devpath = wrap_and_call( |
543 | 'cloudinit.config.cc_resizefs.util', |
544 | {'is_container': {'return_value': False}}, |
545 | - is_device_path_writable_block, 'overlayroot', info, LOG) |
546 | - self.assertFalse(is_writable) |
547 | + maybe_get_writable_device_path, 'overlayroot', info, LOG) |
548 | + self.assertIsNone(devpath) |
549 | self.assertIn( |
550 | "Not attempting to resize devpath 'overlayroot'", |
551 | self.logs.getvalue()) |
552 | |
553 | - def test_is_device_path_writable_block_warns_missing_cmdline_root(self): |
554 | + def test_maybe_get_writable_device_path_warns_missing_cmdline_root(self): |
555 | """When root does not exist isn't in the cmdline, log warning.""" |
556 | info = 'does not matter' |
557 | |
558 | @@ -190,43 +191,43 @@ class TestIsDevicePathWritableBlock(CiTestCase): |
559 | exists_mock_path = 'cloudinit.config.cc_resizefs.os.path.exists' |
560 | with mock.patch(exists_mock_path) as m_exists: |
561 | m_exists.return_value = False |
562 | - is_writable = wrap_and_call( |
563 | + devpath = wrap_and_call( |
564 | 'cloudinit.config.cc_resizefs.util', |
565 | {'is_container': {'return_value': False}, |
566 | 'get_mount_info': {'side_effect': fake_mount_info}, |
567 | 'get_cmdline': {'return_value': 'BOOT_IMAGE=/vmlinuz.efi'}}, |
568 | - is_device_path_writable_block, '/dev/root', info, LOG) |
569 | - self.assertFalse(is_writable) |
570 | + maybe_get_writable_device_path, '/dev/root', info, LOG) |
571 | + self.assertIsNone(devpath) |
572 | logs = self.logs.getvalue() |
573 | self.assertIn("WARNING: Unable to find device '/dev/root'", logs) |
574 | |
575 | - def test_is_device_path_writable_block_does_not_exist(self): |
576 | + def test_maybe_get_writable_device_path_does_not_exist(self): |
577 | """When devpath does not exist, a warning is logged.""" |
578 | info = 'dev=/I/dont/exist mnt_point=/ path=/dev/none' |
579 | - is_writable = wrap_and_call( |
580 | + devpath = wrap_and_call( |
581 | 'cloudinit.config.cc_resizefs.util', |
582 | {'is_container': {'return_value': False}}, |
583 | - is_device_path_writable_block, '/I/dont/exist', info, LOG) |
584 | - self.assertFalse(is_writable) |
585 | + maybe_get_writable_device_path, '/I/dont/exist', info, LOG) |
586 | + self.assertIsNone(devpath) |
587 | self.assertIn( |
588 | "WARNING: Device '/I/dont/exist' did not exist." |
589 | ' cannot resize: %s' % info, |
590 | self.logs.getvalue()) |
591 | |
592 | - def test_is_device_path_writable_block_does_not_exist_in_container(self): |
593 | + def test_maybe_get_writable_device_path_does_not_exist_in_container(self): |
594 | """When devpath does not exist in a container, log a debug message.""" |
595 | info = 'dev=/I/dont/exist mnt_point=/ path=/dev/none' |
596 | - is_writable = wrap_and_call( |
597 | + devpath = wrap_and_call( |
598 | 'cloudinit.config.cc_resizefs.util', |
599 | {'is_container': {'return_value': True}}, |
600 | - is_device_path_writable_block, '/I/dont/exist', info, LOG) |
601 | - self.assertFalse(is_writable) |
602 | + maybe_get_writable_device_path, '/I/dont/exist', info, LOG) |
603 | + self.assertIsNone(devpath) |
604 | self.assertIn( |
605 | "DEBUG: Device '/I/dont/exist' did not exist in container." |
606 | ' cannot resize: %s' % info, |
607 | self.logs.getvalue()) |
608 | |
609 | - def test_is_device_path_writable_block_raises_oserror(self): |
610 | + def test_maybe_get_writable_device_path_raises_oserror(self): |
611 | """When unexpected OSError is raises by os.stat it is reraised.""" |
612 | info = 'dev=/I/dont/exist mnt_point=/ path=/dev/none' |
613 | with self.assertRaises(OSError) as context_manager: |
614 | @@ -234,41 +235,63 @@ class TestIsDevicePathWritableBlock(CiTestCase): |
615 | 'cloudinit.config.cc_resizefs', |
616 | {'util.is_container': {'return_value': True}, |
617 | 'os.stat': {'side_effect': OSError('Something unexpected')}}, |
618 | - is_device_path_writable_block, '/I/dont/exist', info, LOG) |
619 | + maybe_get_writable_device_path, '/I/dont/exist', info, LOG) |
620 | self.assertEqual( |
621 | 'Something unexpected', str(context_manager.exception)) |
622 | |
623 | - def test_is_device_path_writable_block_non_block(self): |
624 | + def test_maybe_get_writable_device_path_non_block(self): |
625 | """When device is not a block device, emit warning return False.""" |
626 | fake_devpath = self.tmp_path('dev/readwrite') |
627 | util.write_file(fake_devpath, '', mode=0o600) # read-write |
628 | info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath) |
629 | |
630 | - is_writable = wrap_and_call( |
631 | + devpath = wrap_and_call( |
632 | 'cloudinit.config.cc_resizefs.util', |
633 | {'is_container': {'return_value': False}}, |
634 | - is_device_path_writable_block, fake_devpath, info, LOG) |
635 | - self.assertFalse(is_writable) |
636 | + maybe_get_writable_device_path, fake_devpath, info, LOG) |
637 | + self.assertIsNone(devpath) |
638 | self.assertIn( |
639 | "WARNING: device '{0}' not a block device. cannot resize".format( |
640 | fake_devpath), |
641 | self.logs.getvalue()) |
642 | |
643 | - def test_is_device_path_writable_block_non_block_on_container(self): |
644 | + def test_maybe_get_writable_device_path_non_block_on_container(self): |
645 | """When device is non-block device in container, emit debug log.""" |
646 | fake_devpath = self.tmp_path('dev/readwrite') |
647 | util.write_file(fake_devpath, '', mode=0o600) # read-write |
648 | info = 'dev=/dev/root mnt_point=/ path={0}'.format(fake_devpath) |
649 | |
650 | - is_writable = wrap_and_call( |
651 | + devpath = wrap_and_call( |
652 | 'cloudinit.config.cc_resizefs.util', |
653 | {'is_container': {'return_value': True}}, |
654 | - is_device_path_writable_block, fake_devpath, info, LOG) |
655 | - self.assertFalse(is_writable) |
656 | + maybe_get_writable_device_path, fake_devpath, info, LOG) |
657 | + self.assertIsNone(devpath) |
658 | self.assertIn( |
659 | "DEBUG: device '{0}' not a block device in container." |
660 | ' cannot resize'.format(fake_devpath), |
661 | self.logs.getvalue()) |
662 | |
663 | + def test_maybe_get_writable_device_path_returns_cmdline_root(self): |
664 | + """When root device is UUID in kernel commandline, update devpath.""" |
665 | + # XXX Long-term we want to use FilesystemMocking test to avoid |
666 | + # touching os.stat. |
667 | + FakeStat = namedtuple( |
668 | + 'FakeStat', ['st_mode', 'st_size', 'st_mtime']) # minimal def. |
669 | + info = 'dev=/dev/root mnt_point=/ path=/does/not/matter' |
670 | + devpath = wrap_and_call( |
671 | + 'cloudinit.config.cc_resizefs', |
672 | + {'util.get_cmdline': {'return_value': 'asdf root=UUID=my-uuid'}, |
673 | + 'util.is_container': False, |
674 | + 'os.path.exists': False, # /dev/root doesn't exist |
675 | + 'os.stat': { |
676 | + 'return_value': FakeStat(25008, 0, 1)} # char block device |
677 | + }, |
678 | + maybe_get_writable_device_path, '/dev/root', info, LOG) |
679 | + self.assertEqual('/dev/disk/by-uuid/my-uuid', devpath) |
680 | + self.assertIn( |
681 | + "DEBUG: Converted /dev/root to '/dev/disk/by-uuid/my-uuid'" |
682 | + " per kernel cmdline", |
683 | + self.logs.getvalue()) |
684 | + |
685 | |
686 | # vi: ts=4 expandtab |
687 | diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py |
688 | index b8fc893..648573f 100644 |
689 | --- a/tests/unittests/test_handler/test_schema.py |
690 | +++ b/tests/unittests/test_handler/test_schema.py |
691 | @@ -4,11 +4,12 @@ from cloudinit.config.schema import ( |
692 | CLOUD_CONFIG_HEADER, SchemaValidationError, annotated_cloudconfig_file, |
693 | get_schema_doc, get_schema, validate_cloudconfig_file, |
694 | validate_cloudconfig_schema, main) |
695 | -from cloudinit.util import write_file |
696 | +from cloudinit.util import subp, write_file |
697 | |
698 | from cloudinit.tests.helpers import CiTestCase, mock, skipIf |
699 | |
700 | from copy import copy |
701 | +import os |
702 | from six import StringIO |
703 | from textwrap import dedent |
704 | from yaml import safe_load |
705 | @@ -364,4 +365,38 @@ class MainTest(CiTestCase): |
706 | self.assertIn( |
707 | 'Valid cloud-config file {0}'.format(myyaml), m_stdout.getvalue()) |
708 | |
709 | + |
710 | +class CloudTestsIntegrationTest(CiTestCase): |
711 | + """Validate all cloud-config yaml schema provided in integration tests. |
712 | + |
713 | + It is less expensive to have unittests validate schema of all cloud-config |
714 | + yaml provided to integration tests, than to run an integration test which |
715 | + raises Warnings or errors on invalid cloud-config schema. |
716 | + """ |
717 | + |
718 | + @skipIf(_missing_jsonschema_dep, "No python-jsonschema dependency") |
719 | + def test_all_integration_test_cloud_config_schema(self): |
720 | + """Validate schema of cloud_tests yaml files looking for warnings.""" |
721 | + schema = get_schema() |
722 | + testsdir = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) |
723 | + integration_testdir = os.path.sep.join( |
724 | + [testsdir, 'cloud_tests', 'testcases']) |
725 | + errors = [] |
726 | + out, _ = subp(['find', integration_testdir, '-name', '*yaml']) |
727 | + for filename in out.splitlines(): |
728 | + test_cfg = safe_load(open(filename)) |
729 | + cloud_config = test_cfg.get('cloud_config') |
730 | + if cloud_config: |
731 | + cloud_config = safe_load( |
732 | + cloud_config.replace("#cloud-config\n", "")) |
733 | + try: |
734 | + validate_cloudconfig_schema( |
735 | + cloud_config, schema, strict=True) |
736 | + except SchemaValidationError as e: |
737 | + errors.append( |
738 | + '{0}: {1}'.format( |
739 | + filename, e)) |
740 | + if errors: |
741 | + raise AssertionError(', '.join(errors)) |
742 | + |
743 | # vi: ts=4 expandtab syntax=python |
744 | diff --git a/tools/read-dependencies b/tools/read-dependencies |
745 | index 2a64868..421f470 100755 |
746 | --- a/tools/read-dependencies |
747 | +++ b/tools/read-dependencies |
748 | @@ -30,9 +30,35 @@ DISTRO_PKG_TYPE_MAP = { |
749 | 'suse': 'suse' |
750 | } |
751 | |
752 | -DISTRO_INSTALL_PKG_CMD = { |
753 | +MAYBE_RELIABLE_YUM_INSTALL = [ |
754 | + 'sh', '-c', |
755 | + """ |
756 | + error() { echo "$@" 1>&2; } |
757 | + n=0; max=10; |
758 | + bcmd="yum install --downloadonly --assumeyes --setopt=keepcache=1" |
759 | + while n=$(($n+1)); do |
760 | + error ":: running $bcmd $* [$n/$max]" |
761 | + $bcmd "$@" |
762 | + r=$? |
763 | + [ $r -eq 0 ] && break |
764 | + [ $n -ge $max ] && { error "gave up on $bcmd"; exit $r; } |
765 | + nap=$(($n*5)) |
766 | + error ":: failed [$r] ($n/$max). sleeping $nap." |
767 | + sleep $nap |
768 | + done |
769 | + error ":: running yum install --cacheonly --assumeyes $*" |
770 | + yum install --cacheonly --assumeyes "$@" |
771 | + """, |
772 | + 'reliable-yum-install'] |
773 | + |
774 | +DRY_DISTRO_INSTALL_PKG_CMD = { |
775 | 'centos': ['yum', 'install', '--assumeyes'], |
776 | 'redhat': ['yum', 'install', '--assumeyes'], |
777 | +} |
778 | + |
779 | +DISTRO_INSTALL_PKG_CMD = { |
780 | + 'centos': MAYBE_RELIABLE_YUM_INSTALL, |
781 | + 'redhat': MAYBE_RELIABLE_YUM_INSTALL, |
782 | 'debian': ['apt', 'install', '-y'], |
783 | 'ubuntu': ['apt', 'install', '-y'], |
784 | 'opensuse': ['zypper', 'install'], |
785 | @@ -80,8 +106,8 @@ def get_parser(): |
786 | help='Additionally install continuous integration system packages ' |
787 | 'required for build and test automation.') |
788 | parser.add_argument( |
789 | - '-v', '--python-version', type=str, dest='python_version', default=None, |
790 | - choices=["2", "3"], |
791 | + '-v', '--python-version', type=str, dest='python_version', |
792 | + default=None, choices=["2", "3"], |
793 | help='Override the version of python we want to generate system ' |
794 | 'package dependencies for. Defaults to the version of python ' |
795 | 'this script is called with') |
796 | @@ -219,10 +245,15 @@ def pkg_install(pkg_list, distro, test_distro=False, dry_run=False): |
797 | '(dryrun)' if dry_run else '', ' '.join(pkg_list))) |
798 | install_cmd = [] |
799 | if dry_run: |
800 | - install_cmd.append('echo') |
801 | + install_cmd.append('echo') |
802 | if os.geteuid() != 0: |
803 | install_cmd.append('sudo') |
804 | - install_cmd.extend(DISTRO_INSTALL_PKG_CMD[distro]) |
805 | + |
806 | + cmd = DISTRO_INSTALL_PKG_CMD[distro] |
807 | + if dry_run and distro in DRY_DISTRO_INSTALL_PKG_CMD: |
808 | + cmd = DRY_DISTRO_INSTALL_PKG_CMD[distro] |
809 | + install_cmd.extend(cmd) |
810 | + |
811 | if distro in ['centos', 'redhat']: |
812 | # CentOS and Redhat need epel-release to access oauthlib and jsonschema |
813 | subprocess.check_call(install_cmd + ['epel-release']) |
814 | diff --git a/tools/run-centos b/tools/run-centos |
815 | index d44d514..d58ef3e 100755 |
816 | --- a/tools/run-centos |
817 | +++ b/tools/run-centos |
818 | @@ -123,7 +123,22 @@ prep() { |
819 | return 0 |
820 | fi |
821 | error "Installing prep packages: ${needed}" |
822 | - yum install --assumeyes ${needed} |
823 | + set -- $needed |
824 | + local n max r |
825 | + n=0; max=10; |
826 | + bcmd="yum install --downloadonly --assumeyes --setopt=keepcache=1" |
827 | + while n=$(($n+1)); do |
828 | + error ":: running $bcmd $* [$n/$max]" |
829 | + $bcmd "$@" |
830 | + r=$? |
831 | + [ $r -eq 0 ] && break |
832 | + [ $n -ge $max ] && { error "gave up on $bcmd"; exit $r; } |
833 | + nap=$(($n*5)) |
834 | + error ":: failed [$r] ($n/$max). sleeping $nap." |
835 | + sleep $nap |
836 | + done |
837 | + error ":: running yum install --cacheonly --assumeyes $*" |
838 | + yum install --cacheonly --assumeyes "$@" |
839 | } |
840 | |
841 | start_container() { |
842 | @@ -153,6 +168,7 @@ start_container() { |
843 | if [ ! -z "${http_proxy-}" ]; then |
844 | debug 1 "configuring proxy ${http_proxy}" |
845 | inside "$name" sh -c "echo proxy=$http_proxy >> /etc/yum.conf" |
846 | + inside "$name" sed -i s/enabled=1/enabled=0/ /etc/yum/pluginconf.d/fastestmirror.conf |
847 | fi |
848 | } |
849 |
PASSED: Continuous integration, rev:a9968540aa2 73bee10b0ca0401 33c01aa3459793 /jenkins. ubuntu. com/server/ job/cloud- init-ci/ 434/
https:/
Executed test runs:
SUCCESS: Checkout
SUCCESS: Unit & Style Tests
SUCCESS: Ubuntu LTS: Build
SUCCESS: Ubuntu LTS: Integration
SUCCESS: MAAS Compatability Testing
IN_PROGRESS: Declarative: Post Actions
Click here to trigger a rebuild: /jenkins. ubuntu. com/server/ job/cloud- init-ci/ 434/rebuild
https:/