Merge ~oddbloke/cloud-init/+git/cloud-init:ubuntu/devel into cloud-init:ubuntu/devel
- Git
- lp:~oddbloke/cloud-init/+git/cloud-init
- ubuntu/devel
- Merge into ubuntu/devel
Proposed by
Dan Watkins
Status: | Merged |
---|---|
Merged at revision: | ba7b33b48d5dfc9f786f55778877922625cac00a |
Proposed branch: | ~oddbloke/cloud-init/+git/cloud-init:ubuntu/devel |
Merge into: | cloud-init:ubuntu/devel |
Diff against target: |
1177 lines (+863/-34) 18 files modified
cloudinit/apport.py (+1/-0) cloudinit/config/cc_set_passwords.py (+34/-19) cloudinit/config/cc_ssh.py (+55/-0) cloudinit/config/tests/test_ssh.py (+166/-0) cloudinit/settings.py (+1/-0) cloudinit/sources/DataSourceExoscale.py (+258/-0) cloudinit/sources/DataSourceGCE.py (+20/-2) cloudinit/sources/__init__.py (+10/-0) cloudinit/url_helper.py (+5/-4) debian/changelog (+12/-0) debian/cloud-init.templates (+3/-3) doc/rtd/conf.py (+0/-5) doc/rtd/topics/datasources.rst (+1/-0) doc/rtd/topics/datasources/exoscale.rst (+68/-0) tests/unittests/test_datasource/test_common.py (+2/-0) tests/unittests/test_datasource/test_exoscale.py (+203/-0) tests/unittests/test_datasource/test_gce.py (+18/-0) tools/ds-identify (+6/-1) |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Ryan Harper | Approve | ||
Review via email: mp+371135@code.launchpad.net |
Commit message
Description of the change
To post a comment you must log in.
Revision history for this message
Ryan Harper (raharper) : | # |
review:
Approve
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | diff --git a/cloudinit/apport.py b/cloudinit/apport.py |
2 | index 22cb7fd..003ff1f 100644 |
3 | --- a/cloudinit/apport.py |
4 | +++ b/cloudinit/apport.py |
5 | @@ -23,6 +23,7 @@ KNOWN_CLOUD_NAMES = [ |
6 | 'CloudStack', |
7 | 'DigitalOcean', |
8 | 'GCE - Google Compute Engine', |
9 | + 'Exoscale', |
10 | 'Hetzner Cloud', |
11 | 'IBM - (aka SoftLayer or BlueMix)', |
12 | 'LXD', |
13 | diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py |
14 | index 4585e4d..cf9b5ab 100755 |
15 | --- a/cloudinit/config/cc_set_passwords.py |
16 | +++ b/cloudinit/config/cc_set_passwords.py |
17 | @@ -9,27 +9,40 @@ |
18 | """ |
19 | Set Passwords |
20 | ------------- |
21 | -**Summary:** Set user passwords |
22 | - |
23 | -Set system passwords and enable or disable ssh password authentication. |
24 | -The ``chpasswd`` config key accepts a dictionary containing a single one of two |
25 | -keys, either ``expire`` or ``list``. If ``expire`` is specified and is set to |
26 | -``false``, then the ``password`` global config key is used as the password for |
27 | -all user accounts. If the ``expire`` key is specified and is set to ``true`` |
28 | -then user passwords will be expired, preventing the default system passwords |
29 | -from being used. |
30 | - |
31 | -If the ``list`` key is provided, a list of |
32 | -``username:password`` pairs can be specified. The usernames specified |
33 | -must already exist on the system, or have been created using the |
34 | -``cc_users_groups`` module. A password can be randomly generated using |
35 | -``username:RANDOM`` or ``username:R``. A hashed password can be specified |
36 | -using ``username:$6$salt$hash``. Password ssh authentication can be |
37 | -enabled, disabled, or left to system defaults using ``ssh_pwauth``. |
38 | +**Summary:** Set user passwords and enable/disable SSH password authentication |
39 | + |
40 | +This module consumes three top-level config keys: ``ssh_pwauth``, ``chpasswd`` |
41 | +and ``password``. |
42 | + |
43 | +The ``ssh_pwauth`` config key determines whether or not sshd will be configured |
44 | +to accept password authentication. True values will enable password auth, |
45 | +false values will disable password auth, and the literal string ``unchanged`` |
46 | +will leave it unchanged. Setting no value will also leave the current setting |
47 | +on-disk unchanged. |
48 | + |
49 | +The ``chpasswd`` config key accepts a dictionary containing either or both of |
50 | +``expire`` and ``list``. |
51 | + |
52 | +If the ``list`` key is provided, it should contain a list of |
53 | +``username:password`` pairs. This can be either a YAML list (of strings), or a |
54 | +multi-line string with one pair per line. Each user will have the |
55 | +corresponding password set. A password can be randomly generated by specifying |
56 | +``RANDOM`` or ``R`` as a user's password. A hashed password, created by a tool |
57 | +like ``mkpasswd``, can be specified; a regex |
58 | +(``r'\\$(1|2a|2y|5|6)(\\$.+){2}'``) is used to determine if a password value |
59 | +should be treated as a hash. |
60 | |
61 | .. note:: |
62 | - if using ``expire: true`` then a ssh authkey should be specified or it may |
63 | - not be possible to login to the system |
64 | + The users specified must already exist on the system. Users will have been |
65 | + created by the ``cc_users_groups`` module at this point. |
66 | + |
67 | +By default, all users on the system will have their passwords expired (meaning |
68 | +that they will have to be reset the next time the user logs in). To disable |
69 | +this behaviour, set ``expire`` under ``chpasswd`` to a false value. |
70 | + |
71 | +If a ``list`` of user/password pairs is not specified under ``chpasswd``, then |
72 | +the value of the ``password`` config key will be used to set the default user's |
73 | +password. |
74 | |
75 | **Internal name:** ``cc_set_passwords`` |
76 | |
77 | @@ -160,6 +173,8 @@ def handle(_name, cfg, cloud, log, args): |
78 | hashed_users = [] |
79 | randlist = [] |
80 | users = [] |
81 | + # N.B. This regex is included in the documentation (i.e. the module |
82 | + # docstring), so any changes to it should be reflected there. |
83 | prog = re.compile(r'\$(1|2a|2y|5|6)(\$.+){2}') |
84 | for line in plist: |
85 | u, p = line.split(':', 1) |
86 | diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py |
87 | index f8f7cb3..53f6939 100755 |
88 | --- a/cloudinit/config/cc_ssh.py |
89 | +++ b/cloudinit/config/cc_ssh.py |
90 | @@ -91,6 +91,9 @@ public keys. |
91 | ssh_authorized_keys: |
92 | - ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAGEA3FSyQwBI6Z+nCSjUU ... |
93 | - ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA3I7VUf2l5gSn5uavROsc5HRDpZ ... |
94 | + ssh_publish_hostkeys: |
95 | + enabled: <true/false> (Defaults to true) |
96 | + blacklist: <list of key types> (Defaults to [dsa]) |
97 | """ |
98 | |
99 | import glob |
100 | @@ -104,6 +107,10 @@ from cloudinit import util |
101 | |
102 | GENERATE_KEY_NAMES = ['rsa', 'dsa', 'ecdsa', 'ed25519'] |
103 | KEY_FILE_TPL = '/etc/ssh/ssh_host_%s_key' |
104 | +PUBLISH_HOST_KEYS = True |
105 | +# Don't publish the dsa hostkey by default since OpenSSH recommends not using |
106 | +# it. |
107 | +HOST_KEY_PUBLISH_BLACKLIST = ['dsa'] |
108 | |
109 | CONFIG_KEY_TO_FILE = {} |
110 | PRIV_TO_PUB = {} |
111 | @@ -176,6 +183,23 @@ def handle(_name, cfg, cloud, log, _args): |
112 | util.logexc(log, "Failed generating key type %s to " |
113 | "file %s", keytype, keyfile) |
114 | |
115 | + if "ssh_publish_hostkeys" in cfg: |
116 | + host_key_blacklist = util.get_cfg_option_list( |
117 | + cfg["ssh_publish_hostkeys"], "blacklist", |
118 | + HOST_KEY_PUBLISH_BLACKLIST) |
119 | + publish_hostkeys = util.get_cfg_option_bool( |
120 | + cfg["ssh_publish_hostkeys"], "enabled", PUBLISH_HOST_KEYS) |
121 | + else: |
122 | + host_key_blacklist = HOST_KEY_PUBLISH_BLACKLIST |
123 | + publish_hostkeys = PUBLISH_HOST_KEYS |
124 | + |
125 | + if publish_hostkeys: |
126 | + hostkeys = get_public_host_keys(blacklist=host_key_blacklist) |
127 | + try: |
128 | + cloud.datasource.publish_host_keys(hostkeys) |
129 | + except Exception as e: |
130 | + util.logexc(log, "Publishing host keys failed!") |
131 | + |
132 | try: |
133 | (users, _groups) = ug_util.normalize_users_groups(cfg, cloud.distro) |
134 | (user, _user_config) = ug_util.extract_default(users) |
135 | @@ -209,4 +233,35 @@ def apply_credentials(keys, user, disable_root, disable_root_opts): |
136 | |
137 | ssh_util.setup_user_keys(keys, 'root', options=key_prefix) |
138 | |
139 | + |
140 | +def get_public_host_keys(blacklist=None): |
141 | + """Read host keys from /etc/ssh/*.pub files and return them as a list. |
142 | + |
143 | + @param blacklist: List of key types to ignore. e.g. ['dsa', 'rsa'] |
144 | + @returns: List of keys, each formatted as a two-element tuple. |
145 | + e.g. [('ssh-rsa', 'AAAAB3Nz...'), ('ssh-ed25519', 'AAAAC3Nx...')] |
146 | + """ |
147 | + public_key_file_tmpl = '%s.pub' % (KEY_FILE_TPL,) |
148 | + key_list = [] |
149 | + blacklist_files = [] |
150 | + if blacklist: |
151 | + # Convert blacklist to filenames: |
152 | + # 'dsa' -> '/etc/ssh/ssh_host_dsa_key.pub' |
153 | + blacklist_files = [public_key_file_tmpl % (key_type,) |
154 | + for key_type in blacklist] |
155 | + # Get list of public key files and filter out blacklisted files. |
156 | + file_list = [hostfile for hostfile |
157 | + in glob.glob(public_key_file_tmpl % ('*',)) |
158 | + if hostfile not in blacklist_files] |
159 | + |
160 | + # Read host key files, retrieve first two fields as a tuple and |
161 | + # append that tuple to key_list. |
162 | + for file_name in file_list: |
163 | + file_contents = util.load_file(file_name) |
164 | + key_data = file_contents.split() |
165 | + if key_data and len(key_data) > 1: |
166 | + key_list.append(tuple(key_data[:2])) |
167 | + return key_list |
168 | + |
169 | + |
170 | # vi: ts=4 expandtab |
171 | diff --git a/cloudinit/config/tests/test_ssh.py b/cloudinit/config/tests/test_ssh.py |
172 | index c8a4271..e778984 100644 |
173 | --- a/cloudinit/config/tests/test_ssh.py |
174 | +++ b/cloudinit/config/tests/test_ssh.py |
175 | @@ -1,5 +1,6 @@ |
176 | # This file is part of cloud-init. See LICENSE file for license information. |
177 | |
178 | +import os.path |
179 | |
180 | from cloudinit.config import cc_ssh |
181 | from cloudinit import ssh_util |
182 | @@ -12,6 +13,25 @@ MODPATH = "cloudinit.config.cc_ssh." |
183 | class TestHandleSsh(CiTestCase): |
184 | """Test cc_ssh handling of ssh config.""" |
185 | |
186 | + def _publish_hostkey_test_setup(self): |
187 | + self.test_hostkeys = { |
188 | + 'dsa': ('ssh-dss', 'AAAAB3NzaC1kc3MAAACB'), |
189 | + 'ecdsa': ('ecdsa-sha2-nistp256', 'AAAAE2VjZ'), |
190 | + 'ed25519': ('ssh-ed25519', 'AAAAC3NzaC1lZDI'), |
191 | + 'rsa': ('ssh-rsa', 'AAAAB3NzaC1yc2EAAA'), |
192 | + } |
193 | + self.test_hostkey_files = [] |
194 | + hostkey_tmpdir = self.tmp_dir() |
195 | + for key_type in ['dsa', 'ecdsa', 'ed25519', 'rsa']: |
196 | + key_data = self.test_hostkeys[key_type] |
197 | + filename = 'ssh_host_%s_key.pub' % key_type |
198 | + filepath = os.path.join(hostkey_tmpdir, filename) |
199 | + self.test_hostkey_files.append(filepath) |
200 | + with open(filepath, 'w') as f: |
201 | + f.write(' '.join(key_data)) |
202 | + |
203 | + cc_ssh.KEY_FILE_TPL = os.path.join(hostkey_tmpdir, 'ssh_host_%s_key') |
204 | + |
205 | def test_apply_credentials_with_user(self, m_setup_keys): |
206 | """Apply keys for the given user and root.""" |
207 | keys = ["key1"] |
208 | @@ -64,6 +84,7 @@ class TestHandleSsh(CiTestCase): |
209 | # Mock os.path.exits to True to short-circuit the key writing logic |
210 | m_path_exists.return_value = True |
211 | m_nug.return_value = ([], {}) |
212 | + cc_ssh.PUBLISH_HOST_KEYS = False |
213 | cloud = self.tmp_cloud( |
214 | distro='ubuntu', metadata={'public-keys': keys}) |
215 | cc_ssh.handle("name", cfg, cloud, None, None) |
216 | @@ -149,3 +170,148 @@ class TestHandleSsh(CiTestCase): |
217 | self.assertEqual([mock.call(set(keys), user), |
218 | mock.call(set(keys), "root", options="")], |
219 | m_setup_keys.call_args_list) |
220 | + |
221 | + @mock.patch(MODPATH + "glob.glob") |
222 | + @mock.patch(MODPATH + "ug_util.normalize_users_groups") |
223 | + @mock.patch(MODPATH + "os.path.exists") |
224 | + def test_handle_publish_hostkeys_default( |
225 | + self, m_path_exists, m_nug, m_glob, m_setup_keys): |
226 | + """Test handle with various configs for ssh_publish_hostkeys.""" |
227 | + self._publish_hostkey_test_setup() |
228 | + cc_ssh.PUBLISH_HOST_KEYS = True |
229 | + keys = ["key1"] |
230 | + user = "clouduser" |
231 | + # Return no matching keys for first glob, test keys for second. |
232 | + m_glob.side_effect = iter([ |
233 | + [], |
234 | + self.test_hostkey_files, |
235 | + ]) |
236 | + # Mock os.path.exits to True to short-circuit the key writing logic |
237 | + m_path_exists.return_value = True |
238 | + m_nug.return_value = ({user: {"default": user}}, {}) |
239 | + cloud = self.tmp_cloud( |
240 | + distro='ubuntu', metadata={'public-keys': keys}) |
241 | + cloud.datasource.publish_host_keys = mock.Mock() |
242 | + |
243 | + cfg = {} |
244 | + expected_call = [self.test_hostkeys[key_type] for key_type |
245 | + in ['ecdsa', 'ed25519', 'rsa']] |
246 | + cc_ssh.handle("name", cfg, cloud, None, None) |
247 | + self.assertEqual([mock.call(expected_call)], |
248 | + cloud.datasource.publish_host_keys.call_args_list) |
249 | + |
250 | + @mock.patch(MODPATH + "glob.glob") |
251 | + @mock.patch(MODPATH + "ug_util.normalize_users_groups") |
252 | + @mock.patch(MODPATH + "os.path.exists") |
253 | + def test_handle_publish_hostkeys_config_enable( |
254 | + self, m_path_exists, m_nug, m_glob, m_setup_keys): |
255 | + """Test handle with various configs for ssh_publish_hostkeys.""" |
256 | + self._publish_hostkey_test_setup() |
257 | + cc_ssh.PUBLISH_HOST_KEYS = False |
258 | + keys = ["key1"] |
259 | + user = "clouduser" |
260 | + # Return no matching keys for first glob, test keys for second. |
261 | + m_glob.side_effect = iter([ |
262 | + [], |
263 | + self.test_hostkey_files, |
264 | + ]) |
265 | + # Mock os.path.exits to True to short-circuit the key writing logic |
266 | + m_path_exists.return_value = True |
267 | + m_nug.return_value = ({user: {"default": user}}, {}) |
268 | + cloud = self.tmp_cloud( |
269 | + distro='ubuntu', metadata={'public-keys': keys}) |
270 | + cloud.datasource.publish_host_keys = mock.Mock() |
271 | + |
272 | + cfg = {'ssh_publish_hostkeys': {'enabled': True}} |
273 | + expected_call = [self.test_hostkeys[key_type] for key_type |
274 | + in ['ecdsa', 'ed25519', 'rsa']] |
275 | + cc_ssh.handle("name", cfg, cloud, None, None) |
276 | + self.assertEqual([mock.call(expected_call)], |
277 | + cloud.datasource.publish_host_keys.call_args_list) |
278 | + |
279 | + @mock.patch(MODPATH + "glob.glob") |
280 | + @mock.patch(MODPATH + "ug_util.normalize_users_groups") |
281 | + @mock.patch(MODPATH + "os.path.exists") |
282 | + def test_handle_publish_hostkeys_config_disable( |
283 | + self, m_path_exists, m_nug, m_glob, m_setup_keys): |
284 | + """Test handle with various configs for ssh_publish_hostkeys.""" |
285 | + self._publish_hostkey_test_setup() |
286 | + cc_ssh.PUBLISH_HOST_KEYS = True |
287 | + keys = ["key1"] |
288 | + user = "clouduser" |
289 | + # Return no matching keys for first glob, test keys for second. |
290 | + m_glob.side_effect = iter([ |
291 | + [], |
292 | + self.test_hostkey_files, |
293 | + ]) |
294 | + # Mock os.path.exits to True to short-circuit the key writing logic |
295 | + m_path_exists.return_value = True |
296 | + m_nug.return_value = ({user: {"default": user}}, {}) |
297 | + cloud = self.tmp_cloud( |
298 | + distro='ubuntu', metadata={'public-keys': keys}) |
299 | + cloud.datasource.publish_host_keys = mock.Mock() |
300 | + |
301 | + cfg = {'ssh_publish_hostkeys': {'enabled': False}} |
302 | + cc_ssh.handle("name", cfg, cloud, None, None) |
303 | + self.assertFalse(cloud.datasource.publish_host_keys.call_args_list) |
304 | + cloud.datasource.publish_host_keys.assert_not_called() |
305 | + |
306 | + @mock.patch(MODPATH + "glob.glob") |
307 | + @mock.patch(MODPATH + "ug_util.normalize_users_groups") |
308 | + @mock.patch(MODPATH + "os.path.exists") |
309 | + def test_handle_publish_hostkeys_config_blacklist( |
310 | + self, m_path_exists, m_nug, m_glob, m_setup_keys): |
311 | + """Test handle with various configs for ssh_publish_hostkeys.""" |
312 | + self._publish_hostkey_test_setup() |
313 | + cc_ssh.PUBLISH_HOST_KEYS = True |
314 | + keys = ["key1"] |
315 | + user = "clouduser" |
316 | + # Return no matching keys for first glob, test keys for second. |
317 | + m_glob.side_effect = iter([ |
318 | + [], |
319 | + self.test_hostkey_files, |
320 | + ]) |
321 | + # Mock os.path.exits to True to short-circuit the key writing logic |
322 | + m_path_exists.return_value = True |
323 | + m_nug.return_value = ({user: {"default": user}}, {}) |
324 | + cloud = self.tmp_cloud( |
325 | + distro='ubuntu', metadata={'public-keys': keys}) |
326 | + cloud.datasource.publish_host_keys = mock.Mock() |
327 | + |
328 | + cfg = {'ssh_publish_hostkeys': {'enabled': True, |
329 | + 'blacklist': ['dsa', 'rsa']}} |
330 | + expected_call = [self.test_hostkeys[key_type] for key_type |
331 | + in ['ecdsa', 'ed25519']] |
332 | + cc_ssh.handle("name", cfg, cloud, None, None) |
333 | + self.assertEqual([mock.call(expected_call)], |
334 | + cloud.datasource.publish_host_keys.call_args_list) |
335 | + |
336 | + @mock.patch(MODPATH + "glob.glob") |
337 | + @mock.patch(MODPATH + "ug_util.normalize_users_groups") |
338 | + @mock.patch(MODPATH + "os.path.exists") |
339 | + def test_handle_publish_hostkeys_empty_blacklist( |
340 | + self, m_path_exists, m_nug, m_glob, m_setup_keys): |
341 | + """Test handle with various configs for ssh_publish_hostkeys.""" |
342 | + self._publish_hostkey_test_setup() |
343 | + cc_ssh.PUBLISH_HOST_KEYS = True |
344 | + keys = ["key1"] |
345 | + user = "clouduser" |
346 | + # Return no matching keys for first glob, test keys for second. |
347 | + m_glob.side_effect = iter([ |
348 | + [], |
349 | + self.test_hostkey_files, |
350 | + ]) |
351 | + # Mock os.path.exits to True to short-circuit the key writing logic |
352 | + m_path_exists.return_value = True |
353 | + m_nug.return_value = ({user: {"default": user}}, {}) |
354 | + cloud = self.tmp_cloud( |
355 | + distro='ubuntu', metadata={'public-keys': keys}) |
356 | + cloud.datasource.publish_host_keys = mock.Mock() |
357 | + |
358 | + cfg = {'ssh_publish_hostkeys': {'enabled': True, |
359 | + 'blacklist': []}} |
360 | + expected_call = [self.test_hostkeys[key_type] for key_type |
361 | + in ['dsa', 'ecdsa', 'ed25519', 'rsa']] |
362 | + cc_ssh.handle("name", cfg, cloud, None, None) |
363 | + self.assertEqual([mock.call(expected_call)], |
364 | + cloud.datasource.publish_host_keys.call_args_list) |
365 | diff --git a/cloudinit/settings.py b/cloudinit/settings.py |
366 | index b1ebaad..2060d81 100644 |
367 | --- a/cloudinit/settings.py |
368 | +++ b/cloudinit/settings.py |
369 | @@ -39,6 +39,7 @@ CFG_BUILTIN = { |
370 | 'Hetzner', |
371 | 'IBMCloud', |
372 | 'Oracle', |
373 | + 'Exoscale', |
374 | # At the end to act as a 'catch' when none of the above work... |
375 | 'None', |
376 | ], |
377 | diff --git a/cloudinit/sources/DataSourceExoscale.py b/cloudinit/sources/DataSourceExoscale.py |
378 | new file mode 100644 |
379 | index 0000000..52e7f6f |
380 | --- /dev/null |
381 | +++ b/cloudinit/sources/DataSourceExoscale.py |
382 | @@ -0,0 +1,258 @@ |
383 | +# Author: Mathieu Corbin <mathieu.corbin@exoscale.com> |
384 | +# Author: Christopher Glass <christopher.glass@exoscale.com> |
385 | +# |
386 | +# This file is part of cloud-init. See LICENSE file for license information. |
387 | + |
388 | +from cloudinit import ec2_utils as ec2 |
389 | +from cloudinit import log as logging |
390 | +from cloudinit import sources |
391 | +from cloudinit import url_helper |
392 | +from cloudinit import util |
393 | + |
394 | +LOG = logging.getLogger(__name__) |
395 | + |
396 | +METADATA_URL = "http://169.254.169.254" |
397 | +API_VERSION = "1.0" |
398 | +PASSWORD_SERVER_PORT = 8080 |
399 | + |
400 | +URL_TIMEOUT = 10 |
401 | +URL_RETRIES = 6 |
402 | + |
403 | +EXOSCALE_DMI_NAME = "Exoscale" |
404 | + |
405 | +BUILTIN_DS_CONFIG = { |
406 | + # We run the set password config module on every boot in order to enable |
407 | + # resetting the instance's password via the exoscale console (and a |
408 | + # subsequent instance reboot). |
409 | + 'cloud_config_modules': [["set-passwords", "always"]] |
410 | +} |
411 | + |
412 | + |
413 | +class DataSourceExoscale(sources.DataSource): |
414 | + |
415 | + dsname = 'Exoscale' |
416 | + |
417 | + def __init__(self, sys_cfg, distro, paths): |
418 | + super(DataSourceExoscale, self).__init__(sys_cfg, distro, paths) |
419 | + LOG.debug("Initializing the Exoscale datasource") |
420 | + |
421 | + self.metadata_url = self.ds_cfg.get('metadata_url', METADATA_URL) |
422 | + self.api_version = self.ds_cfg.get('api_version', API_VERSION) |
423 | + self.password_server_port = int( |
424 | + self.ds_cfg.get('password_server_port', PASSWORD_SERVER_PORT)) |
425 | + self.url_timeout = self.ds_cfg.get('timeout', URL_TIMEOUT) |
426 | + self.url_retries = self.ds_cfg.get('retries', URL_RETRIES) |
427 | + |
428 | + self.extra_config = BUILTIN_DS_CONFIG |
429 | + |
430 | + def wait_for_metadata_service(self): |
431 | + """Wait for the metadata service to be reachable.""" |
432 | + |
433 | + metadata_url = "{}/{}/meta-data/instance-id".format( |
434 | + self.metadata_url, self.api_version) |
435 | + |
436 | + url = url_helper.wait_for_url( |
437 | + urls=[metadata_url], |
438 | + max_wait=self.url_max_wait, |
439 | + timeout=self.url_timeout, |
440 | + status_cb=LOG.critical) |
441 | + |
442 | + return bool(url) |
443 | + |
444 | + def crawl_metadata(self): |
445 | + """ |
446 | + Crawl the metadata service when available. |
447 | + |
448 | + @returns: Dictionary of crawled metadata content. |
449 | + """ |
450 | + metadata_ready = util.log_time( |
451 | + logfunc=LOG.info, |
452 | + msg='waiting for the metadata service', |
453 | + func=self.wait_for_metadata_service) |
454 | + |
455 | + if not metadata_ready: |
456 | + return {} |
457 | + |
458 | + return read_metadata(self.metadata_url, self.api_version, |
459 | + self.password_server_port, self.url_timeout, |
460 | + self.url_retries) |
461 | + |
462 | + def _get_data(self): |
463 | + """Fetch the user data, the metadata and the VM password |
464 | + from the metadata service. |
465 | + |
466 | + Please refer to the datasource documentation for details on how the |
467 | + metadata server and password server are crawled. |
468 | + """ |
469 | + if not self._is_platform_viable(): |
470 | + return False |
471 | + |
472 | + data = util.log_time( |
473 | + logfunc=LOG.debug, |
474 | + msg='Crawl of metadata service', |
475 | + func=self.crawl_metadata) |
476 | + |
477 | + if not data: |
478 | + return False |
479 | + |
480 | + self.userdata_raw = data['user-data'] |
481 | + self.metadata = data['meta-data'] |
482 | + password = data.get('password') |
483 | + |
484 | + password_config = {} |
485 | + if password: |
486 | + # Since we have a password, let's make sure we are allowed to use |
487 | + # it by allowing ssh_pwauth. |
488 | + # The password module's default behavior is to leave the |
489 | + # configuration as-is in this regard, so that means it will either |
490 | + # leave the password always disabled if no password is ever set, or |
491 | + # leave the password login enabled if we set it once. |
492 | + password_config = { |
493 | + 'ssh_pwauth': True, |
494 | + 'password': password, |
495 | + 'chpasswd': { |
496 | + 'expire': False, |
497 | + }, |
498 | + } |
499 | + |
500 | + # builtin extra_config overrides password_config |
501 | + self.extra_config = util.mergemanydict( |
502 | + [self.extra_config, password_config]) |
503 | + |
504 | + return True |
505 | + |
506 | + def get_config_obj(self): |
507 | + return self.extra_config |
508 | + |
509 | + def _is_platform_viable(self): |
510 | + return util.read_dmi_data('system-product-name').startswith( |
511 | + EXOSCALE_DMI_NAME) |
512 | + |
513 | + |
514 | +# Used to match classes to dependencies |
515 | +datasources = [ |
516 | + (DataSourceExoscale, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), |
517 | +] |
518 | + |
519 | + |
520 | +# Return a list of data sources that match this set of dependencies |
521 | +def get_datasource_list(depends): |
522 | + return sources.list_from_depends(depends, datasources) |
523 | + |
524 | + |
525 | +def get_password(metadata_url=METADATA_URL, |
526 | + api_version=API_VERSION, |
527 | + password_server_port=PASSWORD_SERVER_PORT, |
528 | + url_timeout=URL_TIMEOUT, |
529 | + url_retries=URL_RETRIES): |
530 | + """Obtain the VM's password if set. |
531 | + |
532 | + Once fetched the password is marked saved. Future calls to this method may |
533 | + return empty string or 'saved_password'.""" |
534 | + password_url = "{}:{}/{}/".format(metadata_url, password_server_port, |
535 | + api_version) |
536 | + response = url_helper.read_file_or_url( |
537 | + password_url, |
538 | + ssl_details=None, |
539 | + headers={"DomU_Request": "send_my_password"}, |
540 | + timeout=url_timeout, |
541 | + retries=url_retries) |
542 | + password = response.contents.decode('utf-8') |
543 | + # the password is empty or already saved |
544 | + # Note: the original metadata server would answer an additional |
545 | + # 'bad_request' status, but the Exoscale implementation does not. |
546 | + if password in ['', 'saved_password']: |
547 | + return None |
548 | + # save the password |
549 | + url_helper.read_file_or_url( |
550 | + password_url, |
551 | + ssl_details=None, |
552 | + headers={"DomU_Request": "saved_password"}, |
553 | + timeout=url_timeout, |
554 | + retries=url_retries) |
555 | + return password |
556 | + |
557 | + |
558 | +def read_metadata(metadata_url=METADATA_URL, |
559 | + api_version=API_VERSION, |
560 | + password_server_port=PASSWORD_SERVER_PORT, |
561 | + url_timeout=URL_TIMEOUT, |
562 | + url_retries=URL_RETRIES): |
563 | + """Query the metadata server and return the retrieved data.""" |
564 | + crawled_metadata = {} |
565 | + crawled_metadata['_metadata_api_version'] = api_version |
566 | + try: |
567 | + crawled_metadata['user-data'] = ec2.get_instance_userdata( |
568 | + api_version, |
569 | + metadata_url, |
570 | + timeout=url_timeout, |
571 | + retries=url_retries) |
572 | + crawled_metadata['meta-data'] = ec2.get_instance_metadata( |
573 | + api_version, |
574 | + metadata_url, |
575 | + timeout=url_timeout, |
576 | + retries=url_retries) |
577 | + except Exception as e: |
578 | + util.logexc(LOG, "failed reading from metadata url %s (%s)", |
579 | + metadata_url, e) |
580 | + return {} |
581 | + |
582 | + try: |
583 | + crawled_metadata['password'] = get_password( |
584 | + api_version=api_version, |
585 | + metadata_url=metadata_url, |
586 | + password_server_port=password_server_port, |
587 | + url_retries=url_retries, |
588 | + url_timeout=url_timeout) |
589 | + except Exception as e: |
590 | + util.logexc(LOG, "failed to read from password server url %s:%s (%s)", |
591 | + metadata_url, password_server_port, e) |
592 | + |
593 | + return crawled_metadata |
594 | + |
595 | + |
596 | +if __name__ == "__main__": |
597 | + import argparse |
598 | + |
599 | + parser = argparse.ArgumentParser(description='Query Exoscale Metadata') |
600 | + parser.add_argument( |
601 | + "--endpoint", |
602 | + metavar="URL", |
603 | + help="The url of the metadata service.", |
604 | + default=METADATA_URL) |
605 | + parser.add_argument( |
606 | + "--version", |
607 | + metavar="VERSION", |
608 | + help="The version of the metadata endpoint to query.", |
609 | + default=API_VERSION) |
610 | + parser.add_argument( |
611 | + "--retries", |
612 | + metavar="NUM", |
613 | + type=int, |
614 | + help="The number of retries querying the endpoint.", |
615 | + default=URL_RETRIES) |
616 | + parser.add_argument( |
617 | + "--timeout", |
618 | + metavar="NUM", |
619 | + type=int, |
620 | + help="The time in seconds to wait before timing out.", |
621 | + default=URL_TIMEOUT) |
622 | + parser.add_argument( |
623 | + "--password-port", |
624 | + metavar="PORT", |
625 | + type=int, |
626 | + help="The port on which the password endpoint listens", |
627 | + default=PASSWORD_SERVER_PORT) |
628 | + |
629 | + args = parser.parse_args() |
630 | + |
631 | + data = read_metadata( |
632 | + metadata_url=args.endpoint, |
633 | + api_version=args.version, |
634 | + password_server_port=args.password_port, |
635 | + url_timeout=args.timeout, |
636 | + url_retries=args.retries) |
637 | + |
638 | + print(util.json_dumps(data)) |
639 | + |
640 | +# vi: ts=4 expandtab |
641 | diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py |
642 | index d816262..6cbfbba 100644 |
643 | --- a/cloudinit/sources/DataSourceGCE.py |
644 | +++ b/cloudinit/sources/DataSourceGCE.py |
645 | @@ -18,10 +18,13 @@ LOG = logging.getLogger(__name__) |
646 | MD_V1_URL = 'http://metadata.google.internal/computeMetadata/v1/' |
647 | BUILTIN_DS_CONFIG = {'metadata_url': MD_V1_URL} |
648 | REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname') |
649 | +GUEST_ATTRIBUTES_URL = ('http://metadata.google.internal/computeMetadata/' |
650 | + 'v1/instance/guest-attributes') |
651 | +HOSTKEY_NAMESPACE = 'hostkeys' |
652 | +HEADERS = {'Metadata-Flavor': 'Google'} |
653 | |
654 | |
655 | class GoogleMetadataFetcher(object): |
656 | - headers = {'Metadata-Flavor': 'Google'} |
657 | |
658 | def __init__(self, metadata_address): |
659 | self.metadata_address = metadata_address |
660 | @@ -32,7 +35,7 @@ class GoogleMetadataFetcher(object): |
661 | url = self.metadata_address + path |
662 | if is_recursive: |
663 | url += '/?recursive=True' |
664 | - resp = url_helper.readurl(url=url, headers=self.headers) |
665 | + resp = url_helper.readurl(url=url, headers=HEADERS) |
666 | except url_helper.UrlError as exc: |
667 | msg = "url %s raised exception %s" |
668 | LOG.debug(msg, path, exc) |
669 | @@ -90,6 +93,10 @@ class DataSourceGCE(sources.DataSource): |
670 | public_keys_data = self.metadata['public-keys-data'] |
671 | return _parse_public_keys(public_keys_data, self.default_user) |
672 | |
673 | + def publish_host_keys(self, hostkeys): |
674 | + for key in hostkeys: |
675 | + _write_host_key_to_guest_attributes(*key) |
676 | + |
677 | def get_hostname(self, fqdn=False, resolve_ip=False, metadata_only=False): |
678 | # GCE has long FDQN's and has asked for short hostnames. |
679 | return self.metadata['local-hostname'].split('.')[0] |
680 | @@ -103,6 +110,17 @@ class DataSourceGCE(sources.DataSource): |
681 | return self.availability_zone.rsplit('-', 1)[0] |
682 | |
683 | |
684 | +def _write_host_key_to_guest_attributes(key_type, key_value): |
685 | + url = '%s/%s/%s' % (GUEST_ATTRIBUTES_URL, HOSTKEY_NAMESPACE, key_type) |
686 | + key_value = key_value.encode('utf-8') |
687 | + resp = url_helper.readurl(url=url, data=key_value, headers=HEADERS, |
688 | + request_method='PUT', check_status=False) |
689 | + if resp.ok(): |
690 | + LOG.debug('Wrote %s host key to guest attributes.', key_type) |
691 | + else: |
692 | + LOG.debug('Unable to write %s host key to guest attributes.', key_type) |
693 | + |
694 | + |
695 | def _has_expired(public_key): |
696 | # Check whether an SSH key is expired. Public key input is a single SSH |
697 | # public key in the GCE specific key format documented here: |
698 | diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py |
699 | index c2baccd..a319322 100644 |
700 | --- a/cloudinit/sources/__init__.py |
701 | +++ b/cloudinit/sources/__init__.py |
702 | @@ -491,6 +491,16 @@ class DataSource(object): |
703 | def get_public_ssh_keys(self): |
704 | return normalize_pubkey_data(self.metadata.get('public-keys')) |
705 | |
706 | + def publish_host_keys(self, hostkeys): |
707 | + """Publish the public SSH host keys (found in /etc/ssh/*.pub). |
708 | + |
709 | + @param hostkeys: List of host key tuples (key_type, key_value), |
710 | + where key_type is the first field in the public key file |
711 | + (e.g. 'ssh-rsa') and key_value is the key itself |
712 | + (e.g. 'AAAAB3NzaC1y...'). |
713 | + """ |
714 | + pass |
715 | + |
716 | def _remap_device(self, short_name): |
717 | # LP: #611137 |
718 | # the metadata service may believe that devices are named 'sda' |
719 | diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py |
720 | index 0af0d9e..44ee61d 100644 |
721 | --- a/cloudinit/url_helper.py |
722 | +++ b/cloudinit/url_helper.py |
723 | @@ -199,18 +199,19 @@ def _get_ssl_args(url, ssl_details): |
724 | def readurl(url, data=None, timeout=None, retries=0, sec_between=1, |
725 | headers=None, headers_cb=None, ssl_details=None, |
726 | check_status=True, allow_redirects=True, exception_cb=None, |
727 | - session=None, infinite=False, log_req_resp=True): |
728 | + session=None, infinite=False, log_req_resp=True, |
729 | + request_method=None): |
730 | url = _cleanurl(url) |
731 | req_args = { |
732 | 'url': url, |
733 | } |
734 | req_args.update(_get_ssl_args(url, ssl_details)) |
735 | req_args['allow_redirects'] = allow_redirects |
736 | - req_args['method'] = 'GET' |
737 | + if not request_method: |
738 | + request_method = 'POST' if data else 'GET' |
739 | + req_args['method'] = request_method |
740 | if timeout is not None: |
741 | req_args['timeout'] = max(float(timeout), 0) |
742 | - if data: |
743 | - req_args['method'] = 'POST' |
744 | # It doesn't seem like config |
745 | # was added in older library versions (or newer ones either), thus we |
746 | # need to manually do the retries if it wasn't... |
747 | diff --git a/debian/changelog b/debian/changelog |
748 | index 671dad7..2cda24c 100644 |
749 | --- a/debian/changelog |
750 | +++ b/debian/changelog |
751 | @@ -1,3 +1,15 @@ |
752 | +cloud-init (19.2-9-g15584720-0ubuntu1) eoan; urgency=medium |
753 | + |
754 | + * New upstream snapshot. |
755 | + - Add support for publishing host keys to GCE guest attributes |
756 | + [Rick Wright] |
757 | + - New data source for the Exoscale.com cloud platform [Chris Glass] |
758 | + - doc: remove intersphinx extension |
759 | + - cc_set_passwords: rewrite documentation (LP: #1838794) |
760 | + * d/cloud-init.templates: add Exoscale data source |
761 | + |
762 | + -- Daniel Watkins <oddbloke@ubuntu.com> Fri, 09 Aug 2019 13:57:28 -0400 |
763 | + |
764 | cloud-init (19.2-5-g496aaa94-0ubuntu1) eoan; urgency=medium |
765 | |
766 | * New upstream snapshot. |
767 | diff --git a/debian/cloud-init.templates b/debian/cloud-init.templates |
768 | index ece53a0..e5efdad 100644 |
769 | --- a/debian/cloud-init.templates |
770 | +++ b/debian/cloud-init.templates |
771 | @@ -1,8 +1,8 @@ |
772 | Template: cloud-init/datasources |
773 | Type: multiselect |
774 | -Default: NoCloud, ConfigDrive, OpenNebula, DigitalOcean, Azure, AltCloud, OVF, MAAS, GCE, OpenStack, CloudSigma, SmartOS, Bigstep, Scaleway, AliYun, Ec2, CloudStack, Hetzner, IBMCloud, Oracle, None |
775 | -Choices-C: NoCloud, ConfigDrive, OpenNebula, DigitalOcean, Azure, AltCloud, OVF, MAAS, GCE, OpenStack, CloudSigma, SmartOS, Bigstep, Scaleway, AliYun, Ec2, CloudStack, Hetzner, IBMCloud, Oracle, None |
776 | -Choices: NoCloud: Reads info from /var/lib/cloud/seed only, ConfigDrive: Reads data from Openstack Config Drive, OpenNebula: read from OpenNebula context disk, DigitalOcean: reads data from Droplet datasource, Azure: read from MS Azure cdrom. Requires walinux-agent, AltCloud: config disks for RHEVm and vSphere, OVF: Reads data from OVF Transports, MAAS: Reads data from Ubuntu MAAS, GCE: google compute metadata service, OpenStack: native openstack metadata service, CloudSigma: metadata over serial for cloudsigma.com, SmartOS: Read from SmartOS metadata service, Bigstep: Bigstep metadata service, Scaleway: Scaleway metadata service, AliYun: Alibaba metadata service, Ec2: reads data from EC2 Metadata service, CloudStack: Read from CloudStack metadata service, Hetzner: Hetzner Cloud, IBMCloud: IBM Cloud. Previously softlayer or bluemix., Oracle: Oracle Compute Infrastructure, None: Failsafe datasource |
777 | +Default: NoCloud, ConfigDrive, OpenNebula, DigitalOcean, Azure, AltCloud, OVF, MAAS, GCE, OpenStack, CloudSigma, SmartOS, Bigstep, Scaleway, AliYun, Ec2, CloudStack, Hetzner, IBMCloud, Oracle, Exoscale, None |
778 | +Choices-C: NoCloud, ConfigDrive, OpenNebula, DigitalOcean, Azure, AltCloud, OVF, MAAS, GCE, OpenStack, CloudSigma, SmartOS, Bigstep, Scaleway, AliYun, Ec2, CloudStack, Hetzner, IBMCloud, Oracle, Exoscale, None |
779 | +Choices: NoCloud: Reads info from /var/lib/cloud/seed only, ConfigDrive: Reads data from Openstack Config Drive, OpenNebula: read from OpenNebula context disk, DigitalOcean: reads data from Droplet datasource, Azure: read from MS Azure cdrom. Requires walinux-agent, AltCloud: config disks for RHEVm and vSphere, OVF: Reads data from OVF Transports, MAAS: Reads data from Ubuntu MAAS, GCE: google compute metadata service, OpenStack: native openstack metadata service, CloudSigma: metadata over serial for cloudsigma.com, SmartOS: Read from SmartOS metadata service, Bigstep: Bigstep metadata service, Scaleway: Scaleway metadata service, AliYun: Alibaba metadata service, Ec2: reads data from EC2 Metadata service, CloudStack: Read from CloudStack metadata service, Hetzner: Hetzner Cloud, IBMCloud: IBM Cloud. Previously softlayer or bluemix., Oracle: Oracle Compute Infrastructure, Exoscale: Exoscale, None: Failsafe datasource |
780 | Description: Which data sources should be searched? |
781 | Cloud-init supports searching different "Data Sources" for information |
782 | that it uses to configure a cloud instance. |
783 | diff --git a/doc/rtd/conf.py b/doc/rtd/conf.py |
784 | index 50eb05c..4174477 100644 |
785 | --- a/doc/rtd/conf.py |
786 | +++ b/doc/rtd/conf.py |
787 | @@ -27,16 +27,11 @@ project = 'Cloud-Init' |
788 | # Add any Sphinx extension module names here, as strings. They can be |
789 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. |
790 | extensions = [ |
791 | - 'sphinx.ext.intersphinx', |
792 | 'sphinx.ext.autodoc', |
793 | 'sphinx.ext.autosectionlabel', |
794 | 'sphinx.ext.viewcode', |
795 | ] |
796 | |
797 | -intersphinx_mapping = { |
798 | - 'sphinx': ('http://sphinx.pocoo.org', None) |
799 | -} |
800 | - |
801 | # The suffix of source filenames. |
802 | source_suffix = '.rst' |
803 | |
804 | diff --git a/doc/rtd/topics/datasources.rst b/doc/rtd/topics/datasources.rst |
805 | index 648c606..2148cd5 100644 |
806 | --- a/doc/rtd/topics/datasources.rst |
807 | +++ b/doc/rtd/topics/datasources.rst |
808 | @@ -155,6 +155,7 @@ Follow for more information. |
809 | datasources/configdrive.rst |
810 | datasources/digitalocean.rst |
811 | datasources/ec2.rst |
812 | + datasources/exoscale.rst |
813 | datasources/maas.rst |
814 | datasources/nocloud.rst |
815 | datasources/opennebula.rst |
816 | diff --git a/doc/rtd/topics/datasources/exoscale.rst b/doc/rtd/topics/datasources/exoscale.rst |
817 | new file mode 100644 |
818 | index 0000000..27aec9c |
819 | --- /dev/null |
820 | +++ b/doc/rtd/topics/datasources/exoscale.rst |
821 | @@ -0,0 +1,68 @@ |
822 | +.. _datasource_exoscale: |
823 | + |
824 | +Exoscale |
825 | +======== |
826 | + |
827 | +This datasource supports reading from the metadata server used on the |
828 | +`Exoscale platform <https://exoscale.com>`_. |
829 | + |
830 | +Use of the Exoscale datasource is recommended to benefit from new features of |
831 | +the Exoscale platform. |
832 | + |
833 | +The datasource relies on the availability of a compatible metadata server |
834 | +(``http://169.254.169.254`` is used by default) and its companion password |
835 | +server, reachable at the same address (by default on port 8080). |
836 | + |
837 | +Crawling of metadata |
838 | +-------------------- |
839 | + |
840 | +The metadata service and password server are crawled slightly differently: |
841 | + |
842 | + * The "metadata service" is crawled every boot. |
843 | + * The password server is also crawled every boot (the Exoscale datasource |
844 | + forces the password module to run with "frequency always"). |
845 | + |
846 | +In the password server case, the following rules apply in order to enable the |
847 | +"restore instance password" functionality: |
848 | + |
849 | + * If a password is returned by the password server, it is then marked "saved" |
850 | + by the cloud-init datasource. Subsequent boots will skip setting the password |
851 | + (the password server will return "saved_password"). |
852 | + * When the instance password is reset (via the Exoscale UI), the password |
853 | + server will return the non-empty password at next boot, therefore causing |
854 | + cloud-init to reset the instance's password. |
855 | + |
856 | +Configuration |
857 | +------------- |
858 | + |
859 | +Users of this datasource are discouraged from changing the default settings |
860 | +unless instructed to by Exoscale support. |
861 | + |
862 | +The following settings are available and can be set for the datasource in system |
863 | +configuration (in `/etc/cloud/cloud.cfg.d/`). |
864 | + |
865 | +The settings available are: |
866 | + |
867 | + * **metadata_url**: The URL for the metadata service (defaults to |
868 | + ``http://169.254.169.254``) |
869 | + * **api_version**: The API version path on which to query the instance metadata |
870 | + (defaults to ``1.0``) |
871 | + * **password_server_port**: The port (on the metadata server) on which the |
872 | + password server listens (defaults to ``8080``). |
873 | + * **timeout**: the timeout value provided to urlopen for each individual http |
874 | + request. (defaults to ``10``) |
875 | + * **retries**: The number of retries that should be done for an http request |
876 | + (defaults to ``6``) |
877 | + |
878 | + |
879 | +An example configuration with the default values is provided below: |
880 | + |
881 | +.. sourcecode:: yaml |
882 | + |
883 | + datasource: |
884 | + Exoscale: |
885 | + metadata_url: "http://169.254.169.254" |
886 | + api_version: "1.0" |
887 | + password_server_port: 8080 |
888 | + timeout: 10 |
889 | + retries: 6 |
890 | diff --git a/tests/unittests/test_datasource/test_common.py b/tests/unittests/test_datasource/test_common.py |
891 | index 2a9cfb2..61a7a76 100644 |
892 | --- a/tests/unittests/test_datasource/test_common.py |
893 | +++ b/tests/unittests/test_datasource/test_common.py |
894 | @@ -13,6 +13,7 @@ from cloudinit.sources import ( |
895 | DataSourceConfigDrive as ConfigDrive, |
896 | DataSourceDigitalOcean as DigitalOcean, |
897 | DataSourceEc2 as Ec2, |
898 | + DataSourceExoscale as Exoscale, |
899 | DataSourceGCE as GCE, |
900 | DataSourceHetzner as Hetzner, |
901 | DataSourceIBMCloud as IBMCloud, |
902 | @@ -53,6 +54,7 @@ DEFAULT_NETWORK = [ |
903 | CloudStack.DataSourceCloudStack, |
904 | DSNone.DataSourceNone, |
905 | Ec2.DataSourceEc2, |
906 | + Exoscale.DataSourceExoscale, |
907 | GCE.DataSourceGCE, |
908 | MAAS.DataSourceMAAS, |
909 | NoCloud.DataSourceNoCloudNet, |
910 | diff --git a/tests/unittests/test_datasource/test_exoscale.py b/tests/unittests/test_datasource/test_exoscale.py |
911 | new file mode 100644 |
912 | index 0000000..350c330 |
913 | --- /dev/null |
914 | +++ b/tests/unittests/test_datasource/test_exoscale.py |
915 | @@ -0,0 +1,203 @@ |
916 | +# Author: Mathieu Corbin <mathieu.corbin@exoscale.com> |
917 | +# Author: Christopher Glass <christopher.glass@exoscale.com> |
918 | +# |
919 | +# This file is part of cloud-init. See LICENSE file for license information. |
920 | +from cloudinit import helpers |
921 | +from cloudinit.sources.DataSourceExoscale import ( |
922 | + API_VERSION, |
923 | + DataSourceExoscale, |
924 | + METADATA_URL, |
925 | + get_password, |
926 | + PASSWORD_SERVER_PORT, |
927 | + read_metadata) |
928 | +from cloudinit.tests.helpers import HttprettyTestCase, mock |
929 | + |
930 | +import httpretty |
931 | +import requests |
932 | + |
933 | + |
934 | +TEST_PASSWORD_URL = "{}:{}/{}/".format(METADATA_URL, |
935 | + PASSWORD_SERVER_PORT, |
936 | + API_VERSION) |
937 | + |
938 | +TEST_METADATA_URL = "{}/{}/meta-data/".format(METADATA_URL, |
939 | + API_VERSION) |
940 | + |
941 | +TEST_USERDATA_URL = "{}/{}/user-data".format(METADATA_URL, |
942 | + API_VERSION) |
943 | + |
944 | + |
945 | +@httpretty.activate |
946 | +class TestDatasourceExoscale(HttprettyTestCase): |
947 | + |
948 | + def setUp(self): |
949 | + super(TestDatasourceExoscale, self).setUp() |
950 | + self.tmp = self.tmp_dir() |
951 | + self.password_url = TEST_PASSWORD_URL |
952 | + self.metadata_url = TEST_METADATA_URL |
953 | + self.userdata_url = TEST_USERDATA_URL |
954 | + |
955 | + def test_password_saved(self): |
956 | + """The password is not set when it is not found |
957 | + in the metadata service.""" |
958 | + httpretty.register_uri(httpretty.GET, |
959 | + self.password_url, |
960 | + body="saved_password") |
961 | + self.assertFalse(get_password()) |
962 | + |
963 | + def test_password_empty(self): |
964 | + """No password is set if the metadata service returns |
965 | + an empty string.""" |
966 | + httpretty.register_uri(httpretty.GET, |
967 | + self.password_url, |
968 | + body="") |
969 | + self.assertFalse(get_password()) |
970 | + |
971 | + def test_password(self): |
972 | + """The password is set to what is found in the metadata |
973 | + service.""" |
974 | + expected_password = "p@ssw0rd" |
975 | + httpretty.register_uri(httpretty.GET, |
976 | + self.password_url, |
977 | + body=expected_password) |
978 | + password = get_password() |
979 | + self.assertEqual(expected_password, password) |
980 | + |
981 | + def test_get_data(self): |
982 | + """The datasource conforms to expected behavior when supplied |
983 | + full test data.""" |
984 | + path = helpers.Paths({'run_dir': self.tmp}) |
985 | + ds = DataSourceExoscale({}, None, path) |
986 | + ds._is_platform_viable = lambda: True |
987 | + expected_password = "p@ssw0rd" |
988 | + expected_id = "12345" |
989 | + expected_hostname = "myname" |
990 | + expected_userdata = "#cloud-config" |
991 | + httpretty.register_uri(httpretty.GET, |
992 | + self.userdata_url, |
993 | + body=expected_userdata) |
994 | + httpretty.register_uri(httpretty.GET, |
995 | + self.password_url, |
996 | + body=expected_password) |
997 | + httpretty.register_uri(httpretty.GET, |
998 | + self.metadata_url, |
999 | + body="instance-id\nlocal-hostname") |
1000 | + httpretty.register_uri(httpretty.GET, |
1001 | + "{}local-hostname".format(self.metadata_url), |
1002 | + body=expected_hostname) |
1003 | + httpretty.register_uri(httpretty.GET, |
1004 | + "{}instance-id".format(self.metadata_url), |
1005 | + body=expected_id) |
1006 | + self.assertTrue(ds._get_data()) |
1007 | + self.assertEqual(ds.userdata_raw.decode("utf-8"), "#cloud-config") |
1008 | + self.assertEqual(ds.metadata, {"instance-id": expected_id, |
1009 | + "local-hostname": expected_hostname}) |
1010 | + self.assertEqual(ds.get_config_obj(), |
1011 | + {'ssh_pwauth': True, |
1012 | + 'password': expected_password, |
1013 | + 'cloud_config_modules': [ |
1014 | + ["set-passwords", "always"]], |
1015 | + 'chpasswd': { |
1016 | + 'expire': False, |
1017 | + }}) |
1018 | + |
1019 | + def test_get_data_saved_password(self): |
1020 | + """The datasource conforms to expected behavior when saved_password is |
1021 | + returned by the password server.""" |
1022 | + path = helpers.Paths({'run_dir': self.tmp}) |
1023 | + ds = DataSourceExoscale({}, None, path) |
1024 | + ds._is_platform_viable = lambda: True |
1025 | + expected_answer = "saved_password" |
1026 | + expected_id = "12345" |
1027 | + expected_hostname = "myname" |
1028 | + expected_userdata = "#cloud-config" |
1029 | + httpretty.register_uri(httpretty.GET, |
1030 | + self.userdata_url, |
1031 | + body=expected_userdata) |
1032 | + httpretty.register_uri(httpretty.GET, |
1033 | + self.password_url, |
1034 | + body=expected_answer) |
1035 | + httpretty.register_uri(httpretty.GET, |
1036 | + self.metadata_url, |
1037 | + body="instance-id\nlocal-hostname") |
1038 | + httpretty.register_uri(httpretty.GET, |
1039 | + "{}local-hostname".format(self.metadata_url), |
1040 | + body=expected_hostname) |
1041 | + httpretty.register_uri(httpretty.GET, |
1042 | + "{}instance-id".format(self.metadata_url), |
1043 | + body=expected_id) |
1044 | + self.assertTrue(ds._get_data()) |
1045 | + self.assertEqual(ds.userdata_raw.decode("utf-8"), "#cloud-config") |
1046 | + self.assertEqual(ds.metadata, {"instance-id": expected_id, |
1047 | + "local-hostname": expected_hostname}) |
1048 | + self.assertEqual(ds.get_config_obj(), |
1049 | + {'cloud_config_modules': [ |
1050 | + ["set-passwords", "always"]]}) |
1051 | + |
1052 | + def test_get_data_no_password(self): |
1053 | + """The datasource conforms to expected behavior when no password is |
1054 | + returned by the password server.""" |
1055 | + path = helpers.Paths({'run_dir': self.tmp}) |
1056 | + ds = DataSourceExoscale({}, None, path) |
1057 | + ds._is_platform_viable = lambda: True |
1058 | + expected_answer = "" |
1059 | + expected_id = "12345" |
1060 | + expected_hostname = "myname" |
1061 | + expected_userdata = "#cloud-config" |
1062 | + httpretty.register_uri(httpretty.GET, |
1063 | + self.userdata_url, |
1064 | + body=expected_userdata) |
1065 | + httpretty.register_uri(httpretty.GET, |
1066 | + self.password_url, |
1067 | + body=expected_answer) |
1068 | + httpretty.register_uri(httpretty.GET, |
1069 | + self.metadata_url, |
1070 | + body="instance-id\nlocal-hostname") |
1071 | + httpretty.register_uri(httpretty.GET, |
1072 | + "{}local-hostname".format(self.metadata_url), |
1073 | + body=expected_hostname) |
1074 | + httpretty.register_uri(httpretty.GET, |
1075 | + "{}instance-id".format(self.metadata_url), |
1076 | + body=expected_id) |
1077 | + self.assertTrue(ds._get_data()) |
1078 | + self.assertEqual(ds.userdata_raw.decode("utf-8"), "#cloud-config") |
1079 | + self.assertEqual(ds.metadata, {"instance-id": expected_id, |
1080 | + "local-hostname": expected_hostname}) |
1081 | + self.assertEqual(ds.get_config_obj(), |
1082 | + {'cloud_config_modules': [ |
1083 | + ["set-passwords", "always"]]}) |
1084 | + |
1085 | + @mock.patch('cloudinit.sources.DataSourceExoscale.get_password') |
1086 | + def test_read_metadata_when_password_server_unreachable(self, m_password): |
1087 | + """The read_metadata function returns partial results in case the |
1088 | + password server (only) is unreachable.""" |
1089 | + expected_id = "12345" |
1090 | + expected_hostname = "myname" |
1091 | + expected_userdata = "#cloud-config" |
1092 | + |
1093 | + m_password.side_effect = requests.Timeout('Fake Connection Timeout') |
1094 | + httpretty.register_uri(httpretty.GET, |
1095 | + self.userdata_url, |
1096 | + body=expected_userdata) |
1097 | + httpretty.register_uri(httpretty.GET, |
1098 | + self.metadata_url, |
1099 | + body="instance-id\nlocal-hostname") |
1100 | + httpretty.register_uri(httpretty.GET, |
1101 | + "{}local-hostname".format(self.metadata_url), |
1102 | + body=expected_hostname) |
1103 | + httpretty.register_uri(httpretty.GET, |
1104 | + "{}instance-id".format(self.metadata_url), |
1105 | + body=expected_id) |
1106 | + |
1107 | + result = read_metadata() |
1108 | + |
1109 | + self.assertIsNone(result.get("password")) |
1110 | + self.assertEqual(result.get("user-data").decode("utf-8"), |
1111 | + expected_userdata) |
1112 | + |
1113 | + def test_non_viable_platform(self): |
1114 | + """The datasource fails fast when the platform is not viable.""" |
1115 | + path = helpers.Paths({'run_dir': self.tmp}) |
1116 | + ds = DataSourceExoscale({}, None, path) |
1117 | + ds._is_platform_viable = lambda: False |
1118 | + self.assertFalse(ds._get_data()) |
1119 | diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py |
1120 | index 41176c6..67744d3 100644 |
1121 | --- a/tests/unittests/test_datasource/test_gce.py |
1122 | +++ b/tests/unittests/test_datasource/test_gce.py |
1123 | @@ -55,6 +55,8 @@ GCE_USER_DATA_TEXT = { |
1124 | HEADERS = {'Metadata-Flavor': 'Google'} |
1125 | MD_URL_RE = re.compile( |
1126 | r'http://metadata.google.internal/computeMetadata/v1/.*') |
1127 | +GUEST_ATTRIBUTES_URL = ('http://metadata.google.internal/computeMetadata/' |
1128 | + 'v1/instance/guest-attributes/hostkeys/') |
1129 | |
1130 | |
1131 | def _set_mock_metadata(gce_meta=None): |
1132 | @@ -341,4 +343,20 @@ class TestDataSourceGCE(test_helpers.HttprettyTestCase): |
1133 | public_key_data, default_user='default') |
1134 | self.assertEqual(sorted(found), sorted(expected)) |
1135 | |
1136 | + @mock.patch("cloudinit.url_helper.readurl") |
1137 | + def test_publish_host_keys(self, m_readurl): |
1138 | + hostkeys = [('ssh-rsa', 'asdfasdf'), |
1139 | + ('ssh-ed25519', 'qwerqwer')] |
1140 | + readurl_expected_calls = [ |
1141 | + mock.call(check_status=False, data=b'asdfasdf', headers=HEADERS, |
1142 | + request_method='PUT', |
1143 | + url='%s%s' % (GUEST_ATTRIBUTES_URL, 'ssh-rsa')), |
1144 | + mock.call(check_status=False, data=b'qwerqwer', headers=HEADERS, |
1145 | + request_method='PUT', |
1146 | + url='%s%s' % (GUEST_ATTRIBUTES_URL, 'ssh-ed25519')), |
1147 | + ] |
1148 | + self.ds.publish_host_keys(hostkeys) |
1149 | + m_readurl.assert_has_calls(readurl_expected_calls, any_order=True) |
1150 | + |
1151 | + |
1152 | # vi: ts=4 expandtab |
1153 | diff --git a/tools/ds-identify b/tools/ds-identify |
1154 | index 0305e36..e0d4865 100755 |
1155 | --- a/tools/ds-identify |
1156 | +++ b/tools/ds-identify |
1157 | @@ -124,7 +124,7 @@ DI_DSNAME="" |
1158 | # be searched if there is no setting found in config. |
1159 | DI_DSLIST_DEFAULT="MAAS ConfigDrive NoCloud AltCloud Azure Bigstep \ |
1160 | CloudSigma CloudStack DigitalOcean AliYun Ec2 GCE OpenNebula OpenStack \ |
1161 | -OVF SmartOS Scaleway Hetzner IBMCloud Oracle" |
1162 | +OVF SmartOS Scaleway Hetzner IBMCloud Oracle Exoscale" |
1163 | DI_DSLIST="" |
1164 | DI_MODE="" |
1165 | DI_ON_FOUND="" |
1166 | @@ -553,6 +553,11 @@ dscheck_CloudStack() { |
1167 | return $DS_NOT_FOUND |
1168 | } |
1169 | |
1170 | +dscheck_Exoscale() { |
1171 | + dmi_product_name_matches "Exoscale*" && return $DS_FOUND |
1172 | + return $DS_NOT_FOUND |
1173 | +} |
1174 | + |
1175 | dscheck_CloudSigma() { |
1176 | # http://paste.ubuntu.com/23624795/ |
1177 | dmi_product_name_matches "CloudSigma" && return $DS_FOUND |