Merge lp:~smoser/cloud-init/trunk.joyent-cleanup into lp:~cloud-init-dev/cloud-init/trunk
- trunk.joyent-cleanup
- Merge into trunk
Status: | Merged |
---|---|
Merged at revision: | 1225 |
Proposed branch: | lp:~smoser/cloud-init/trunk.joyent-cleanup |
Merge into: | lp:~cloud-init-dev/cloud-init/trunk |
Diff against target: |
1140 lines (+544/-349) 2 files modified
cloudinit/sources/DataSourceSmartOS.py (+416/-175) tests/unittests/test_datasource/test_smartos.py (+128/-174) |
To merge this branch: | bzr merge lp:~smoser/cloud-init/trunk.joyent-cleanup |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
cloud-init Commiters | Pending | ||
Review via email: mp+295981@code.launchpad.net |
Commit message
smartos: separate client out of datasource and support network config
This does:
* support reading 'sdc:nics' and converting it to internal
'network-config' format.
* change the datasource to be a local datasource (which is required
to apply networking configuration).
This change requires some rework of init to allow a datasource
to be found locally but not resolve all userdata/metadata until
networking is up. (that is coming)
* support 'list', 'put', and 'delete' in the metadata client.
* add 'get_json' to get known values that have json content.
* add a simple main to datasource demonstrating how to read.
* separate out the "legacy" behavior of having optionally
base64 encoded data into JoyentMetadataL
Note, tests have been dropped that expected datasource.md[key] to be
base64 encoded. Now, the client takes care of decoding that. There
is not any real reason to store the base64 values in the datasource.
Description of the change
- 1230. By Scott Moser
-
assertEquals
- 1231. By Scott Moser
-
return dict not None on get_config_obj
Scott Moser (smoser) wrote : | # |
- 1232. By Scott Moser
-
assertEqual
- 1233. By Scott Moser
-
use constants for kvm and lx-brand
Preview Diff
1 | === modified file 'cloudinit/sources/DataSourceSmartOS.py' | |||
2 | --- cloudinit/sources/DataSourceSmartOS.py 2016-04-12 16:57:50 +0000 | |||
3 | +++ cloudinit/sources/DataSourceSmartOS.py 2016-05-31 17:14:35 +0000 | |||
4 | @@ -32,13 +32,13 @@ | |||
5 | 32 | # http://us-east.manta.joyent.com/jmc/public/mdata/datadict.html | 32 | # http://us-east.manta.joyent.com/jmc/public/mdata/datadict.html |
6 | 33 | # Comments with "@datadictionary" are snippets of the definition | 33 | # Comments with "@datadictionary" are snippets of the definition |
7 | 34 | 34 | ||
8 | 35 | import base64 | ||
9 | 35 | import binascii | 36 | import binascii |
11 | 36 | import contextlib | 37 | import json |
12 | 37 | import os | 38 | import os |
13 | 38 | import random | 39 | import random |
14 | 39 | import re | 40 | import re |
15 | 40 | import socket | 41 | import socket |
16 | 41 | import stat | ||
17 | 42 | 42 | ||
18 | 43 | import serial | 43 | import serial |
19 | 44 | 44 | ||
20 | @@ -64,14 +64,36 @@ | |||
21 | 64 | 'operator-script': ('sdc:operator-script', False), | 64 | 'operator-script': ('sdc:operator-script', False), |
22 | 65 | } | 65 | } |
23 | 66 | 66 | ||
24 | 67 | SMARTOS_ATTRIB_JSON = { | ||
25 | 68 | # Cloud-init Key : (SmartOS Key known JSON) | ||
26 | 69 | 'network-data': 'sdc:nics', | ||
27 | 70 | } | ||
28 | 71 | |||
29 | 72 | SMARTOS_ENV_LX_BRAND = "lx-brand" | ||
30 | 73 | SMARTOS_ENV_KVM = "kvm" | ||
31 | 74 | |||
32 | 67 | DS_NAME = 'SmartOS' | 75 | DS_NAME = 'SmartOS' |
33 | 68 | DS_CFG_PATH = ['datasource', DS_NAME] | 76 | DS_CFG_PATH = ['datasource', DS_NAME] |
34 | 77 | NO_BASE64_DECODE = [ | ||
35 | 78 | 'iptables_disable', | ||
36 | 79 | 'motd_sys_info', | ||
37 | 80 | 'root_authorized_keys', | ||
38 | 81 | 'sdc:datacenter_name', | ||
39 | 82 | 'sdc:uuid' | ||
40 | 83 | 'user-data', | ||
41 | 84 | 'user-script', | ||
42 | 85 | ] | ||
43 | 86 | |||
44 | 87 | METADATA_SOCKFILE = '/native/.zonecontrol/metadata.sock' | ||
45 | 88 | SERIAL_DEVICE = '/dev/ttyS1' | ||
46 | 89 | SERIAL_TIMEOUT = 60 | ||
47 | 90 | |||
48 | 69 | # BUILT-IN DATASOURCE CONFIGURATION | 91 | # BUILT-IN DATASOURCE CONFIGURATION |
49 | 70 | # The following is the built-in configuration. If the values | 92 | # The following is the built-in configuration. If the values |
50 | 71 | # are not set via the system configuration, then these default | 93 | # are not set via the system configuration, then these default |
51 | 72 | # will be used: | 94 | # will be used: |
52 | 73 | # serial_device: which serial device to use for the meta-data | 95 | # serial_device: which serial device to use for the meta-data |
54 | 74 | # seed_timeout: how long to wait on the device | 96 | # serial_timeout: how long to wait on the device |
55 | 75 | # no_base64_decode: values which are not base64 encoded and | 97 | # no_base64_decode: values which are not base64 encoded and |
56 | 76 | # are fetched directly from SmartOS, not meta-data values | 98 | # are fetched directly from SmartOS, not meta-data values |
57 | 77 | # base64_keys: meta-data keys that are delivered in base64 | 99 | # base64_keys: meta-data keys that are delivered in base64 |
58 | @@ -81,16 +103,10 @@ | |||
59 | 81 | # fs_setup: describes how to format the ephemeral drive | 103 | # fs_setup: describes how to format the ephemeral drive |
60 | 82 | # | 104 | # |
61 | 83 | BUILTIN_DS_CONFIG = { | 105 | BUILTIN_DS_CONFIG = { |
72 | 84 | 'serial_device': '/dev/ttyS1', | 106 | 'serial_device': SERIAL_DEVICE, |
73 | 85 | 'metadata_sockfile': '/native/.zonecontrol/metadata.sock', | 107 | 'serial_timeout': SERIAL_TIMEOUT, |
74 | 86 | 'seed_timeout': 60, | 108 | 'metadata_sockfile': METADATA_SOCKFILE, |
75 | 87 | 'no_base64_decode': ['root_authorized_keys', | 109 | 'no_base64_decode': NO_BASE64_DECODE, |
66 | 88 | 'motd_sys_info', | ||
67 | 89 | 'iptables_disable', | ||
68 | 90 | 'user-data', | ||
69 | 91 | 'user-script', | ||
70 | 92 | 'sdc:datacenter_name', | ||
71 | 93 | 'sdc:uuid'], | ||
76 | 94 | 'base64_keys': [], | 110 | 'base64_keys': [], |
77 | 95 | 'base64_all': False, | 111 | 'base64_all': False, |
78 | 96 | 'disk_aliases': {'ephemeral0': '/dev/vdb'}, | 112 | 'disk_aliases': {'ephemeral0': '/dev/vdb'}, |
79 | @@ -154,59 +170,40 @@ | |||
80 | 154 | 170 | ||
81 | 155 | 171 | ||
82 | 156 | class DataSourceSmartOS(sources.DataSource): | 172 | class DataSourceSmartOS(sources.DataSource): |
83 | 173 | _unset = "_unset" | ||
84 | 174 | smartos_environ = _unset | ||
85 | 175 | md_client = _unset | ||
86 | 176 | |||
87 | 157 | def __init__(self, sys_cfg, distro, paths): | 177 | def __init__(self, sys_cfg, distro, paths): |
88 | 158 | sources.DataSource.__init__(self, sys_cfg, distro, paths) | 178 | sources.DataSource.__init__(self, sys_cfg, distro, paths) |
89 | 159 | self.is_smartdc = None | ||
90 | 160 | self.ds_cfg = util.mergemanydict([ | 179 | self.ds_cfg = util.mergemanydict([ |
91 | 161 | self.ds_cfg, | 180 | self.ds_cfg, |
92 | 162 | util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}), | 181 | util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}), |
93 | 163 | BUILTIN_DS_CONFIG]) | 182 | BUILTIN_DS_CONFIG]) |
94 | 164 | 183 | ||
95 | 165 | self.metadata = {} | 184 | self.metadata = {} |
96 | 185 | self.network_data = None | ||
97 | 186 | self._network_config = None | ||
98 | 166 | 187 | ||
99 | 167 | # SDC LX-Brand Zones lack dmidecode (no /dev/mem) but | ||
100 | 168 | # report 'BrandZ virtual linux' as the kernel version | ||
101 | 169 | if os.uname()[3].lower() == 'brandz virtual linux': | ||
102 | 170 | LOG.debug("Host is SmartOS, guest in Zone") | ||
103 | 171 | self.is_smartdc = True | ||
104 | 172 | self.smartos_type = 'lx-brand' | ||
105 | 173 | self.cfg = {} | ||
106 | 174 | self.seed = self.ds_cfg.get("metadata_sockfile") | ||
107 | 175 | else: | ||
108 | 176 | self.is_smartdc = True | ||
109 | 177 | self.smartos_type = 'kvm' | ||
110 | 178 | self.seed = self.ds_cfg.get("serial_device") | ||
111 | 179 | self.cfg = BUILTIN_CLOUD_CONFIG | ||
112 | 180 | self.seed_timeout = self.ds_cfg.get("serial_timeout") | ||
113 | 181 | self.smartos_no_base64 = self.ds_cfg.get('no_base64_decode') | ||
114 | 182 | self.b64_keys = self.ds_cfg.get('base64_keys') | ||
115 | 183 | self.b64_all = self.ds_cfg.get('base64_all') | ||
116 | 184 | self.script_base_d = os.path.join(self.paths.get_cpath("scripts")) | 188 | self.script_base_d = os.path.join(self.paths.get_cpath("scripts")) |
117 | 189 | self.smartos_type = None | ||
118 | 190 | |||
119 | 191 | self._init() | ||
120 | 185 | 192 | ||
121 | 186 | def __str__(self): | 193 | def __str__(self): |
122 | 187 | root = sources.DataSource.__str__(self) | 194 | root = sources.DataSource.__str__(self) |
145 | 188 | return "%s [seed=%s]" % (root, self.seed) | 195 | return "%s [client=%s]" % (root, self.md_client) |
146 | 189 | 196 | ||
147 | 190 | def _get_seed_file_object(self): | 197 | def _init(self): |
148 | 191 | if not self.seed: | 198 | if self.smartos_environ == self._unset: |
149 | 192 | raise AttributeError("seed device is not set") | 199 | self.smartos_type = get_smartos_environ() |
150 | 193 | 200 | ||
151 | 194 | if self.smartos_type == 'lx-brand': | 201 | if self.md_client == self._unset: |
152 | 195 | if not stat.S_ISSOCK(os.stat(self.seed).st_mode): | 202 | self.md_client = jmc_client_factory( |
153 | 196 | LOG.debug("Seed %s is not a socket", self.seed) | 203 | smartos_type=self.smartos_type, |
154 | 197 | return None | 204 | metadata_sockfile=self.ds_cfg['metadata_sockfile'], |
155 | 198 | sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) | 205 | serial_device=self.ds_cfg['serial_device'], |
156 | 199 | sock.connect(self.seed) | 206 | serial_timeout=self.ds_cfg['serial_timeout']) |
135 | 200 | return sock.makefile('rwb') | ||
136 | 201 | else: | ||
137 | 202 | if not stat.S_ISCHR(os.stat(self.seed).st_mode): | ||
138 | 203 | LOG.debug("Seed %s is not a character device") | ||
139 | 204 | return None | ||
140 | 205 | ser = serial.Serial(self.seed, timeout=self.seed_timeout) | ||
141 | 206 | if not ser.isOpen(): | ||
142 | 207 | raise SystemError("Unable to open %s" % self.seed) | ||
143 | 208 | return ser | ||
144 | 209 | return None | ||
157 | 210 | 207 | ||
158 | 211 | def _set_provisioned(self): | 208 | def _set_provisioned(self): |
159 | 212 | '''Mark the instance provisioning state as successful. | 209 | '''Mark the instance provisioning state as successful. |
160 | @@ -225,50 +222,26 @@ | |||
161 | 225 | '/'.join([svc_path, 'provision_success'])) | 222 | '/'.join([svc_path, 'provision_success'])) |
162 | 226 | 223 | ||
163 | 227 | def get_data(self): | 224 | def get_data(self): |
164 | 225 | self._init() | ||
165 | 226 | |||
166 | 228 | md = {} | 227 | md = {} |
167 | 229 | ud = "" | 228 | ud = "" |
168 | 230 | 229 | ||
210 | 231 | if not device_exists(self.seed): | 230 | if not self.smartos_type: |
211 | 232 | LOG.debug("No metadata device '%s' found for SmartOS datasource", | 231 | LOG.debug("Not running on smartos") |
212 | 233 | self.seed) | 232 | return False |
213 | 234 | return False | 233 | |
214 | 235 | 234 | if not self.md_client.exists(): | |
215 | 236 | uname_arch = os.uname()[4] | 235 | LOG.debug("No metadata device '%r' found for SmartOS datasource", |
216 | 237 | if uname_arch.startswith("arm") or uname_arch == "aarch64": | 236 | self.md_client) |
217 | 238 | # Disabling because dmidcode in dmi_data() crashes kvm process | 237 | return False |
218 | 239 | LOG.debug("Disabling SmartOS datasource on arm (LP: #1243287)") | 238 | |
219 | 240 | return False | 239 | for ci_noun, attribute in SMARTOS_ATTRIB_MAP.items(): |
220 | 241 | 240 | smartos_noun, strip = attribute | |
221 | 242 | # SDC KVM instances will provide dmi data, LX-brand does not | 241 | md[ci_noun] = self.md_client.get(smartos_noun, strip=strip) |
222 | 243 | if self.smartos_type == 'kvm': | 242 | |
223 | 244 | dmi_info = dmi_data() | 243 | for ci_noun, smartos_noun in SMARTOS_ATTRIB_JSON.items(): |
224 | 245 | if dmi_info is None: | 244 | md[ci_noun] = self.md_client.get_json(smartos_noun) |
184 | 246 | LOG.debug("No dmidata utility found") | ||
185 | 247 | return False | ||
186 | 248 | |||
187 | 249 | system_type = dmi_info | ||
188 | 250 | if 'smartdc' not in system_type.lower(): | ||
189 | 251 | LOG.debug("Host is not on SmartOS. system_type=%s", | ||
190 | 252 | system_type) | ||
191 | 253 | return False | ||
192 | 254 | LOG.debug("Host is SmartOS, guest in KVM") | ||
193 | 255 | |||
194 | 256 | seed_obj = self._get_seed_file_object() | ||
195 | 257 | if seed_obj is None: | ||
196 | 258 | LOG.debug('Seed file object not found.') | ||
197 | 259 | return False | ||
198 | 260 | with contextlib.closing(seed_obj) as seed: | ||
199 | 261 | b64_keys = self.query('base64_keys', seed, strip=True, b64=False) | ||
200 | 262 | if b64_keys is not None: | ||
201 | 263 | self.b64_keys = [k.strip() for k in str(b64_keys).split(',')] | ||
202 | 264 | |||
203 | 265 | b64_all = self.query('base64_all', seed, strip=True, b64=False) | ||
204 | 266 | if b64_all is not None: | ||
205 | 267 | self.b64_all = util.is_true(b64_all) | ||
206 | 268 | |||
207 | 269 | for ci_noun, attribute in SMARTOS_ATTRIB_MAP.items(): | ||
208 | 270 | smartos_noun, strip = attribute | ||
209 | 271 | md[ci_noun] = self.query(smartos_noun, seed, strip=strip) | ||
225 | 272 | 245 | ||
226 | 273 | # @datadictionary: This key may contain a program that is written | 246 | # @datadictionary: This key may contain a program that is written |
227 | 274 | # to a file in the filesystem of the guest on each boot and then | 247 | # to a file in the filesystem of the guest on each boot and then |
228 | @@ -318,6 +291,7 @@ | |||
229 | 318 | self.metadata = util.mergemanydict([md, self.metadata]) | 291 | self.metadata = util.mergemanydict([md, self.metadata]) |
230 | 319 | self.userdata_raw = ud | 292 | self.userdata_raw = ud |
231 | 320 | self.vendordata_raw = md['vendor-data'] | 293 | self.vendordata_raw = md['vendor-data'] |
232 | 294 | self.network_data = md['network-data'] | ||
233 | 321 | 295 | ||
234 | 322 | self._set_provisioned() | 296 | self._set_provisioned() |
235 | 323 | return True | 297 | return True |
236 | @@ -326,69 +300,20 @@ | |||
237 | 326 | return self.ds_cfg['disk_aliases'].get(name) | 300 | return self.ds_cfg['disk_aliases'].get(name) |
238 | 327 | 301 | ||
239 | 328 | def get_config_obj(self): | 302 | def get_config_obj(self): |
241 | 329 | return self.cfg | 303 | if self.smartos_type == SMARTOS_ENV_KVM: |
242 | 304 | return BUILTIN_CLOUD_CONFIG | ||
243 | 305 | return {} | ||
244 | 330 | 306 | ||
245 | 331 | def get_instance_id(self): | 307 | def get_instance_id(self): |
246 | 332 | return self.metadata['instance-id'] | 308 | return self.metadata['instance-id'] |
247 | 333 | 309 | ||
306 | 334 | def query(self, noun, seed_file, strip=False, default=None, b64=None): | 310 | @property |
307 | 335 | if b64 is None: | 311 | def network_config(self): |
308 | 336 | if noun in self.smartos_no_base64: | 312 | if self._network_config is None: |
309 | 337 | b64 = False | 313 | if self.network_data is not None: |
310 | 338 | elif self.b64_all or noun in self.b64_keys: | 314 | self._network_config = ( |
311 | 339 | b64 = True | 315 | convert_smartos_network_data(self.network_data)) |
312 | 340 | 316 | return self._network_config | |
255 | 341 | return self._query_data(noun, seed_file, strip=strip, | ||
256 | 342 | default=default, b64=b64) | ||
257 | 343 | |||
258 | 344 | def _query_data(self, noun, seed_file, strip=False, | ||
259 | 345 | default=None, b64=None): | ||
260 | 346 | """Makes a request via "GET <NOUN>" | ||
261 | 347 | |||
262 | 348 | In the response, the first line is the status, while subsequent | ||
263 | 349 | lines are is the value. A blank line with a "." is used to | ||
264 | 350 | indicate end of response. | ||
265 | 351 | |||
266 | 352 | If the response is expected to be base64 encoded, then set | ||
267 | 353 | b64encoded to true. Unfortantely, there is no way to know if | ||
268 | 354 | something is 100% encoded, so this method relies on being told | ||
269 | 355 | if the data is base64 or not. | ||
270 | 356 | """ | ||
271 | 357 | |||
272 | 358 | if not noun: | ||
273 | 359 | return False | ||
274 | 360 | |||
275 | 361 | response = JoyentMetadataClient(seed_file).get_metadata(noun) | ||
276 | 362 | |||
277 | 363 | if response is None: | ||
278 | 364 | return default | ||
279 | 365 | |||
280 | 366 | if b64 is None: | ||
281 | 367 | b64 = self._query_data('b64-%s' % noun, seed_file, b64=False, | ||
282 | 368 | default=False, strip=True) | ||
283 | 369 | b64 = util.is_true(b64) | ||
284 | 370 | |||
285 | 371 | resp = None | ||
286 | 372 | if b64 or strip: | ||
287 | 373 | resp = "".join(response).rstrip() | ||
288 | 374 | else: | ||
289 | 375 | resp = "".join(response) | ||
290 | 376 | |||
291 | 377 | if b64: | ||
292 | 378 | try: | ||
293 | 379 | return util.b64d(resp) | ||
294 | 380 | # Bogus input produces different errors in Python 2 and 3; | ||
295 | 381 | # catch both. | ||
296 | 382 | except (TypeError, binascii.Error): | ||
297 | 383 | LOG.warn("Failed base64 decoding key '%s'", noun) | ||
298 | 384 | return resp | ||
299 | 385 | |||
300 | 386 | return resp | ||
301 | 387 | |||
302 | 388 | |||
303 | 389 | def device_exists(device): | ||
304 | 390 | """Symplistic method to determine if the device exists or not""" | ||
305 | 391 | return os.path.exists(device) | ||
313 | 392 | 317 | ||
314 | 393 | 318 | ||
315 | 394 | class JoyentMetadataFetchException(Exception): | 319 | class JoyentMetadataFetchException(Exception): |
316 | @@ -407,8 +332,11 @@ | |||
317 | 407 | r' (?P<body>(?P<request_id>[0-9a-f]+) (?P<status>SUCCESS|NOTFOUND)' | 332 | r' (?P<body>(?P<request_id>[0-9a-f]+) (?P<status>SUCCESS|NOTFOUND)' |
318 | 408 | r'( (?P<payload>.+))?)') | 333 | r'( (?P<payload>.+))?)') |
319 | 409 | 334 | ||
322 | 410 | def __init__(self, metasource): | 335 | def __init__(self, smartos_type=None, fp=None): |
323 | 411 | self.metasource = metasource | 336 | if smartos_type is None: |
324 | 337 | smartos_type = get_smartos_environ() | ||
325 | 338 | self.smartos_type = smartos_type | ||
326 | 339 | self.fp = fp | ||
327 | 412 | 340 | ||
328 | 413 | def _checksum(self, body): | 341 | def _checksum(self, body): |
329 | 414 | return '{0:08x}'.format( | 342 | return '{0:08x}'.format( |
330 | @@ -436,37 +364,227 @@ | |||
331 | 436 | LOG.debug('Value "%s" found.', value) | 364 | LOG.debug('Value "%s" found.', value) |
332 | 437 | return value | 365 | return value |
333 | 438 | 366 | ||
336 | 439 | def get_metadata(self, metadata_key): | 367 | def request(self, rtype, param=None): |
335 | 440 | LOG.debug('Fetching metadata key "%s"...', metadata_key) | ||
337 | 441 | request_id = '{0:08x}'.format(random.randint(0, 0xffffffff)) | 368 | request_id = '{0:08x}'.format(random.randint(0, 0xffffffff)) |
340 | 442 | message_body = '{0} GET {1}'.format(request_id, | 369 | message_body = ' '.join((request_id, rtype,)) |
341 | 443 | util.b64e(metadata_key)) | 370 | if param: |
342 | 371 | message_body += ' ' + base64.b64encode(param.encode()).decode() | ||
343 | 444 | msg = 'V2 {0} {1} {2}\n'.format( | 372 | msg = 'V2 {0} {1} {2}\n'.format( |
344 | 445 | len(message_body), self._checksum(message_body), message_body) | 373 | len(message_body), self._checksum(message_body), message_body) |
345 | 446 | LOG.debug('Writing "%s" to metadata transport.', msg) | 374 | LOG.debug('Writing "%s" to metadata transport.', msg) |
348 | 447 | self.metasource.write(msg.encode('ascii')) | 375 | |
349 | 448 | self.metasource.flush() | 376 | need_close = False |
350 | 377 | if not self.fp: | ||
351 | 378 | self.open_transport() | ||
352 | 379 | need_close = True | ||
353 | 380 | |||
354 | 381 | self.fp.write(msg.encode('ascii')) | ||
355 | 382 | self.fp.flush() | ||
356 | 449 | 383 | ||
357 | 450 | response = bytearray() | 384 | response = bytearray() |
359 | 451 | response.extend(self.metasource.read(1)) | 385 | response.extend(self.fp.read(1)) |
360 | 452 | while response[-1:] != b'\n': | 386 | while response[-1:] != b'\n': |
362 | 453 | response.extend(self.metasource.read(1)) | 387 | response.extend(self.fp.read(1)) |
363 | 388 | |||
364 | 389 | if need_close: | ||
365 | 390 | self.close_transport() | ||
366 | 391 | |||
367 | 454 | response = response.rstrip().decode('ascii') | 392 | response = response.rstrip().decode('ascii') |
368 | 455 | LOG.debug('Read "%s" from metadata transport.', response) | 393 | LOG.debug('Read "%s" from metadata transport.', response) |
369 | 456 | 394 | ||
370 | 457 | if 'SUCCESS' not in response: | 395 | if 'SUCCESS' not in response: |
371 | 458 | return None | 396 | return None |
372 | 459 | 397 | ||
383 | 460 | return self._get_value_from_frame(request_id, response) | 398 | value = self._get_value_from_frame(request_id, response) |
384 | 461 | 399 | return value | |
385 | 462 | 400 | ||
386 | 463 | def dmi_data(): | 401 | def get(self, key, default=None, strip=False): |
387 | 464 | sys_type = util.read_dmi_data("system-product-name") | 402 | result = self.request(rtype='GET', param=key) |
388 | 465 | 403 | if result is None: | |
389 | 466 | if not sys_type: | 404 | return default |
390 | 467 | return None | 405 | if result and strip: |
391 | 468 | 406 | result = result.strip() | |
392 | 469 | return sys_type | 407 | return result |
393 | 408 | |||
394 | 409 | def get_json(self, key, default=None): | ||
395 | 410 | result = self.get(key, default=default) | ||
396 | 411 | if result is None: | ||
397 | 412 | return default | ||
398 | 413 | return json.loads(result) | ||
399 | 414 | |||
400 | 415 | def list(self): | ||
401 | 416 | result = self.request(rtype='KEYS') | ||
402 | 417 | if result: | ||
403 | 418 | result = result.split('\n') | ||
404 | 419 | return result | ||
405 | 420 | |||
406 | 421 | def put(self, key, val): | ||
407 | 422 | param = b' '.join([base64.b64encode(i.encode()) | ||
408 | 423 | for i in (key, val)]).decode() | ||
409 | 424 | return self.request(rtype='PUT', param=param) | ||
410 | 425 | |||
411 | 426 | def delete(self, key): | ||
412 | 427 | return self.request(rtype='DELETE', param=key) | ||
413 | 428 | |||
414 | 429 | def close_transport(self): | ||
415 | 430 | if self.fp: | ||
416 | 431 | self.fp.close() | ||
417 | 432 | self.fp = None | ||
418 | 433 | |||
419 | 434 | def __enter__(self): | ||
420 | 435 | if self.fp: | ||
421 | 436 | return self | ||
422 | 437 | self.open_transport() | ||
423 | 438 | return self | ||
424 | 439 | |||
425 | 440 | def __exit__(self, exc_type, exc_value, traceback): | ||
426 | 441 | self.close_transport() | ||
427 | 442 | return | ||
428 | 443 | |||
429 | 444 | def open_transport(self): | ||
430 | 445 | raise NotImplementedError | ||
431 | 446 | |||
432 | 447 | |||
433 | 448 | class JoyentMetadataSocketClient(JoyentMetadataClient): | ||
434 | 449 | def __init__(self, socketpath): | ||
435 | 450 | self.socketpath = socketpath | ||
436 | 451 | |||
437 | 452 | def open_transport(self): | ||
438 | 453 | sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) | ||
439 | 454 | sock.connect(self.socketpath) | ||
440 | 455 | self.fp = sock.makefile('rwb') | ||
441 | 456 | |||
442 | 457 | def exists(self): | ||
443 | 458 | return os.path.exists(self.socketpath) | ||
444 | 459 | |||
445 | 460 | def __repr__(self): | ||
446 | 461 | return "%s(socketpath=%s)" % (self.__class__.__name__, self.socketpath) | ||
447 | 462 | |||
448 | 463 | |||
449 | 464 | class JoyentMetadataSerialClient(JoyentMetadataClient): | ||
450 | 465 | def __init__(self, device, timeout=10, smartos_type=None): | ||
451 | 466 | super(JoyentMetadataSerialClient, self).__init__(smartos_type) | ||
452 | 467 | self.device = device | ||
453 | 468 | self.timeout = timeout | ||
454 | 469 | |||
455 | 470 | def exists(self): | ||
456 | 471 | return os.path.exists(self.device) | ||
457 | 472 | |||
458 | 473 | def open_transport(self): | ||
459 | 474 | ser = serial.Serial(self.device, timeout=self.timeout) | ||
460 | 475 | if not ser.isOpen(): | ||
461 | 476 | raise SystemError("Unable to open %s" % self.device) | ||
462 | 477 | self.fp = ser | ||
463 | 478 | |||
464 | 479 | def __repr__(self): | ||
465 | 480 | return "%s(device=%s, timeout=%s)" % ( | ||
466 | 481 | self.__class__.__name__, self.device, self.timeout) | ||
467 | 482 | |||
468 | 483 | |||
469 | 484 | class JoyentMetadataLegacySerialClient(JoyentMetadataSerialClient): | ||
470 | 485 | """V1 of the protocol was not safe for all values. | ||
471 | 486 | Thus, we allowed the user to pass values in as base64 encoded. | ||
472 | 487 | Users may still reasonably expect to be able to send base64 data | ||
473 | 488 | and have it transparently decoded. So even though the V2 format is | ||
474 | 489 | now used, and is safe (using base64 itself), we keep legacy support. | ||
475 | 490 | |||
476 | 491 | The way for a user to do this was: | ||
477 | 492 | a.) specify 'base64_keys' key whose value is a comma delimited | ||
478 | 493 | list of keys that were base64 encoded. | ||
479 | 494 | b.) base64_all: string interpreted as a boolean that indicates | ||
480 | 495 | if all keys are base64 encoded. | ||
481 | 496 | c.) set a key named b64-<keyname> with a boolean indicating that | ||
482 | 497 | <keyname> is base64 encoded.""" | ||
483 | 498 | |||
484 | 499 | def __init__(self, device, timeout=10, smartos_type=None): | ||
485 | 500 | s = super(JoyentMetadataLegacySerialClient, self) | ||
486 | 501 | s.__init__(device, timeout, smartos_type) | ||
487 | 502 | self.base64_keys = None | ||
488 | 503 | self.base64_all = None | ||
489 | 504 | |||
490 | 505 | def _init_base64_keys(self, reset=False): | ||
491 | 506 | if reset: | ||
492 | 507 | self.base64_keys = None | ||
493 | 508 | self.base64_all = None | ||
494 | 509 | |||
495 | 510 | keys = None | ||
496 | 511 | if self.base64_all is None: | ||
497 | 512 | keys = self.list() | ||
498 | 513 | if 'base64_all' in keys: | ||
499 | 514 | self.base64_all = util.is_true(self._get("base64_all")) | ||
500 | 515 | else: | ||
501 | 516 | self.base64_all = False | ||
502 | 517 | |||
503 | 518 | if self.base64_all: | ||
504 | 519 | # short circuit if base64_all is true | ||
505 | 520 | return | ||
506 | 521 | |||
507 | 522 | if self.base64_keys is None: | ||
508 | 523 | if keys is None: | ||
509 | 524 | keys = self.list() | ||
510 | 525 | b64_keys = set() | ||
511 | 526 | if 'base64_keys' in keys: | ||
512 | 527 | b64_keys = set(self._get("base64_keys").split(",")) | ||
513 | 528 | |||
514 | 529 | # now add any b64-<keyname> that has a true value | ||
515 | 530 | for key in [k[3:] for k in keys if k.startswith("b64-")]: | ||
516 | 531 | if util.is_true(self._get(key)): | ||
517 | 532 | b64_keys.add(key) | ||
518 | 533 | else: | ||
519 | 534 | if key in b64_keys: | ||
520 | 535 | b64_keys.remove(key) | ||
521 | 536 | |||
522 | 537 | self.base64_keys = b64_keys | ||
523 | 538 | |||
524 | 539 | def _get(self, key, default=None, strip=False): | ||
525 | 540 | return (super(JoyentMetadataLegacySerialClient, self). | ||
526 | 541 | get(key, default=default, strip=strip)) | ||
527 | 542 | |||
528 | 543 | def is_b64_encoded(self, key, reset=False): | ||
529 | 544 | if key in NO_BASE64_DECODE: | ||
530 | 545 | return False | ||
531 | 546 | |||
532 | 547 | self._init_base64_keys(reset=reset) | ||
533 | 548 | if self.base64_all: | ||
534 | 549 | return True | ||
535 | 550 | |||
536 | 551 | return key in self.base64_keys | ||
537 | 552 | |||
538 | 553 | def get(self, key, default=None, strip=False): | ||
539 | 554 | mdefault = object() | ||
540 | 555 | val = self._get(key, strip=False, default=mdefault) | ||
541 | 556 | if val is mdefault: | ||
542 | 557 | return default | ||
543 | 558 | |||
544 | 559 | if self.is_b64_encoded(key): | ||
545 | 560 | try: | ||
546 | 561 | val = base64.b64decode(val.encode()).decode() | ||
547 | 562 | # Bogus input produces different errors in Python 2 and 3 | ||
548 | 563 | except (TypeError, binascii.Error): | ||
549 | 564 | LOG.warn("Failed base64 decoding key '%s': %s", key, val) | ||
550 | 565 | |||
551 | 566 | if strip: | ||
552 | 567 | val = val.strip() | ||
553 | 568 | |||
554 | 569 | return val | ||
555 | 570 | |||
556 | 571 | |||
557 | 572 | def jmc_client_factory( | ||
558 | 573 | smartos_type=None, metadata_sockfile=METADATA_SOCKFILE, | ||
559 | 574 | serial_device=SERIAL_DEVICE, serial_timeout=SERIAL_TIMEOUT, | ||
560 | 575 | uname_version=None): | ||
561 | 576 | |||
562 | 577 | if smartos_type is None: | ||
563 | 578 | smartos_type = get_smartos_environ(uname_version) | ||
564 | 579 | |||
565 | 580 | if smartos_type == SMARTOS_ENV_KVM: | ||
566 | 581 | return JoyentMetadataLegacySerialClient( | ||
567 | 582 | device=serial_device, timeout=serial_timeout, | ||
568 | 583 | smartos_type=smartos_type) | ||
569 | 584 | elif smartos_type == SMARTOS_ENV_LX_BRAND: | ||
570 | 585 | return JoyentMetadataSocketClient(socketpath=metadata_sockfile) | ||
571 | 586 | |||
572 | 587 | raise ValueError("Unknown value for smartos_type: %s" % smartos_type) | ||
573 | 470 | 588 | ||
574 | 471 | 589 | ||
575 | 472 | def write_boot_content(content, content_f, link=None, shebang=False, | 590 | def write_boot_content(content, content_f, link=None, shebang=False, |
576 | @@ -522,15 +640,138 @@ | |||
577 | 522 | util.ensure_dir(os.path.dirname(link)) | 640 | util.ensure_dir(os.path.dirname(link)) |
578 | 523 | os.symlink(content_f, link) | 641 | os.symlink(content_f, link) |
579 | 524 | except IOError as e: | 642 | except IOError as e: |
581 | 525 | util.logexc(LOG, "failed establishing content link", e) | 643 | util.logexc(LOG, "failed establishing content link: %s", e) |
582 | 644 | |||
583 | 645 | |||
584 | 646 | def get_smartos_environ(uname_version=None, product_name=None, | ||
585 | 647 | uname_arch=None): | ||
586 | 648 | uname = os.uname() | ||
587 | 649 | if uname_arch is None: | ||
588 | 650 | uname_arch = uname[4] | ||
589 | 651 | |||
590 | 652 | if uname_arch.startswith("arm") or uname_arch == "aarch64": | ||
591 | 653 | return None | ||
592 | 654 | |||
593 | 655 | # SDC LX-Brand Zones lack dmidecode (no /dev/mem) but | ||
594 | 656 | # report 'BrandZ virtual linux' as the kernel version | ||
595 | 657 | if uname_version is None: | ||
596 | 658 | uname_version = uname[3] | ||
597 | 659 | if uname_version.lower() == 'brandz virtual linux': | ||
598 | 660 | return SMARTOS_ENV_LX_BRAND | ||
599 | 661 | |||
600 | 662 | if product_name is None: | ||
601 | 663 | system_type = util.read_dmi_data("system-product-name") | ||
602 | 664 | else: | ||
603 | 665 | system_type = product_name | ||
604 | 666 | |||
605 | 667 | if system_type and 'smartdc' in system_type.lower(): | ||
606 | 668 | return SMARTOS_ENV_KVM | ||
607 | 669 | |||
608 | 670 | return None | ||
609 | 671 | |||
610 | 672 | |||
611 | 673 | # Covert SMARTOS 'sdc:nics' data to network_config yaml | ||
612 | 674 | def convert_smartos_network_data(network_data=None): | ||
613 | 675 | """Return a dictionary of network_config by parsing provided | ||
614 | 676 | SMARTOS sdc:nics configuration data | ||
615 | 677 | |||
616 | 678 | sdc:nics data is a dictionary of properties of a nic and the ip | ||
617 | 679 | configuration desired. Additional nic dictionaries are appended | ||
618 | 680 | to the list. | ||
619 | 681 | |||
620 | 682 | Converting the format is straightforward though it does include | ||
621 | 683 | duplicate information as well as data which appears to be relevant | ||
622 | 684 | to the hostOS rather than the guest. | ||
623 | 685 | |||
624 | 686 | For each entry in the nics list returned from query sdc:nics, we | ||
625 | 687 | create a type: physical entry, and extract the interface properties: | ||
626 | 688 | 'mac' -> 'mac_address', 'mtu', 'interface' -> 'name'. The remaining | ||
627 | 689 | keys are related to ip configuration. For each ip in the 'ips' list | ||
628 | 690 | we create a subnet entry under 'subnets' pairing the ip to a one in | ||
629 | 691 | the 'gateways' list. | ||
630 | 692 | """ | ||
631 | 693 | |||
632 | 694 | valid_keys = { | ||
633 | 695 | 'physical': [ | ||
634 | 696 | 'mac_address', | ||
635 | 697 | 'mtu', | ||
636 | 698 | 'name', | ||
637 | 699 | 'params', | ||
638 | 700 | 'subnets', | ||
639 | 701 | 'type', | ||
640 | 702 | ], | ||
641 | 703 | 'subnet': [ | ||
642 | 704 | 'address', | ||
643 | 705 | 'broadcast', | ||
644 | 706 | 'dns_nameservers', | ||
645 | 707 | 'dns_search', | ||
646 | 708 | 'gateway', | ||
647 | 709 | 'metric', | ||
648 | 710 | 'netmask', | ||
649 | 711 | 'pointopoint', | ||
650 | 712 | 'routes', | ||
651 | 713 | 'scope', | ||
652 | 714 | 'type', | ||
653 | 715 | ], | ||
654 | 716 | } | ||
655 | 717 | |||
656 | 718 | config = [] | ||
657 | 719 | for nic in network_data: | ||
658 | 720 | cfg = {k: v for k, v in nic.items() | ||
659 | 721 | if k in valid_keys['physical']} | ||
660 | 722 | cfg.update({ | ||
661 | 723 | 'type': 'physical', | ||
662 | 724 | 'name': nic['interface']}) | ||
663 | 725 | if 'mac' in nic: | ||
664 | 726 | cfg.update({'mac_address': nic['mac']}) | ||
665 | 727 | |||
666 | 728 | subnets = [] | ||
667 | 729 | for ip, gw in zip(nic['ips'], nic['gateways']): | ||
668 | 730 | subnet = {k: v for k, v in nic.items() | ||
669 | 731 | if k in valid_keys['subnet']} | ||
670 | 732 | subnet.update({ | ||
671 | 733 | 'type': 'static', | ||
672 | 734 | 'address': ip, | ||
673 | 735 | 'gateway': gw, | ||
674 | 736 | }) | ||
675 | 737 | subnets.append(subnet) | ||
676 | 738 | cfg.update({'subnets': subnets}) | ||
677 | 739 | config.append(cfg) | ||
678 | 740 | |||
679 | 741 | return {'version': 1, 'config': config} | ||
680 | 526 | 742 | ||
681 | 527 | 743 | ||
682 | 528 | # Used to match classes to dependencies | 744 | # Used to match classes to dependencies |
683 | 529 | datasources = [ | 745 | datasources = [ |
685 | 530 | (DataSourceSmartOS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), | 746 | (DataSourceSmartOS, (sources.DEP_FILESYSTEM, )), |
686 | 531 | ] | 747 | ] |
687 | 532 | 748 | ||
688 | 533 | 749 | ||
689 | 534 | # Return a list of data sources that match this set of dependencies | 750 | # Return a list of data sources that match this set of dependencies |
690 | 535 | def get_datasource_list(depends): | 751 | def get_datasource_list(depends): |
691 | 536 | return sources.list_from_depends(depends, datasources) | 752 | return sources.list_from_depends(depends, datasources) |
692 | 753 | |||
693 | 754 | |||
694 | 755 | if __name__ == "__main__": | ||
695 | 756 | import sys | ||
696 | 757 | jmc = jmc_client_factory() | ||
697 | 758 | if len(sys.argv) == 1: | ||
698 | 759 | keys = (list(SMARTOS_ATTRIB_JSON.keys()) + | ||
699 | 760 | list(SMARTOS_ATTRIB_MAP.keys())) | ||
700 | 761 | else: | ||
701 | 762 | keys = sys.argv[1:] | ||
702 | 763 | |||
703 | 764 | data = {} | ||
704 | 765 | for key in keys: | ||
705 | 766 | if key in SMARTOS_ATTRIB_JSON: | ||
706 | 767 | keyname = SMARTOS_ATTRIB_JSON[key] | ||
707 | 768 | data[key] = jmc.get_json(keyname) | ||
708 | 769 | else: | ||
709 | 770 | if key in SMARTOS_ATTRIB_MAP: | ||
710 | 771 | keyname, strip = SMARTOS_ATTRIB_MAP[key] | ||
711 | 772 | else: | ||
712 | 773 | keyname, strip = (key, False) | ||
713 | 774 | val = jmc.get(keyname, strip=strip) | ||
714 | 775 | data[key] = jmc.get(keyname, strip=strip) | ||
715 | 776 | |||
716 | 777 | print(json.dumps(data, indent=1)) | ||
717 | 537 | 778 | ||
718 | === modified file 'tests/unittests/test_datasource/test_smartos.py' | |||
719 | --- tests/unittests/test_datasource/test_smartos.py 2016-05-12 20:43:11 +0000 | |||
720 | +++ tests/unittests/test_datasource/test_smartos.py 2016-05-31 17:14:35 +0000 | |||
721 | @@ -25,6 +25,7 @@ | |||
722 | 25 | from __future__ import print_function | 25 | from __future__ import print_function |
723 | 26 | 26 | ||
724 | 27 | from binascii import crc32 | 27 | from binascii import crc32 |
725 | 28 | import json | ||
726 | 28 | import os | 29 | import os |
727 | 29 | import os.path | 30 | import os.path |
728 | 30 | import re | 31 | import re |
729 | @@ -40,12 +41,49 @@ | |||
730 | 40 | from cloudinit.sources import DataSourceSmartOS | 41 | from cloudinit.sources import DataSourceSmartOS |
731 | 41 | from cloudinit.util import b64e | 42 | from cloudinit.util import b64e |
732 | 42 | 43 | ||
734 | 43 | from .. import helpers | 44 | from ..helpers import mock, FilesystemMockingTestCase, TestCase |
735 | 44 | 45 | ||
740 | 45 | try: | 46 | SDC_NICS = json.loads(""" |
741 | 46 | from unittest import mock | 47 | [ |
742 | 47 | except ImportError: | 48 | { |
743 | 48 | import mock | 49 | "nic_tag": "external", |
744 | 50 | "primary": true, | ||
745 | 51 | "mtu": 1500, | ||
746 | 52 | "model": "virtio", | ||
747 | 53 | "gateway": "8.12.42.1", | ||
748 | 54 | "netmask": "255.255.255.0", | ||
749 | 55 | "ip": "8.12.42.102", | ||
750 | 56 | "network_uuid": "992fc7ce-6aac-4b74-aed6-7b9d2c6c0bfe", | ||
751 | 57 | "gateways": [ | ||
752 | 58 | "8.12.42.1" | ||
753 | 59 | ], | ||
754 | 60 | "vlan_id": 324, | ||
755 | 61 | "mac": "90:b8:d0:f5:e4:f5", | ||
756 | 62 | "interface": "net0", | ||
757 | 63 | "ips": [ | ||
758 | 64 | "8.12.42.102/24" | ||
759 | 65 | ] | ||
760 | 66 | }, | ||
761 | 67 | { | ||
762 | 68 | "nic_tag": "sdc_overlay/16187209", | ||
763 | 69 | "gateway": "192.168.128.1", | ||
764 | 70 | "model": "virtio", | ||
765 | 71 | "mac": "90:b8:d0:a5:ff:cd", | ||
766 | 72 | "netmask": "255.255.252.0", | ||
767 | 73 | "ip": "192.168.128.93", | ||
768 | 74 | "network_uuid": "4cad71da-09bc-452b-986d-03562a03a0a9", | ||
769 | 75 | "gateways": [ | ||
770 | 76 | "192.168.128.1" | ||
771 | 77 | ], | ||
772 | 78 | "vlan_id": 2, | ||
773 | 79 | "mtu": 8500, | ||
774 | 80 | "interface": "net1", | ||
775 | 81 | "ips": [ | ||
776 | 82 | "192.168.128.93/22" | ||
777 | 83 | ] | ||
778 | 84 | } | ||
779 | 85 | ] | ||
780 | 86 | """) | ||
781 | 49 | 87 | ||
782 | 50 | MOCK_RETURNS = { | 88 | MOCK_RETURNS = { |
783 | 51 | 'hostname': 'test-host', | 89 | 'hostname': 'test-host', |
784 | @@ -60,79 +98,66 @@ | |||
785 | 60 | 'sdc:vendor-data': '\n'.join(['VENDOR_DATA', '']), | 98 | 'sdc:vendor-data': '\n'.join(['VENDOR_DATA', '']), |
786 | 61 | 'user-data': '\n'.join(['something', '']), | 99 | 'user-data': '\n'.join(['something', '']), |
787 | 62 | 'user-script': '\n'.join(['/bin/true', '']), | 100 | 'user-script': '\n'.join(['/bin/true', '']), |
788 | 101 | 'sdc:nics': json.dumps(SDC_NICS), | ||
789 | 63 | } | 102 | } |
790 | 64 | 103 | ||
791 | 65 | DMI_DATA_RETURN = 'smartdc' | 104 | DMI_DATA_RETURN = 'smartdc' |
792 | 66 | 105 | ||
793 | 67 | 106 | ||
806 | 68 | def get_mock_client(mockdata): | 107 | class PsuedoJoyentClient(object): |
807 | 69 | class MockMetadataClient(object): | 108 | def __init__(self, data=None): |
808 | 70 | 109 | if data is None: | |
809 | 71 | def __init__(self, serial): | 110 | data = MOCK_RETURNS.copy() |
810 | 72 | pass | 111 | self.data = data |
811 | 73 | 112 | return | |
812 | 74 | def get_metadata(self, metadata_key): | 113 | |
813 | 75 | return mockdata.get(metadata_key) | 114 | def get(self, key, default=None, strip=False): |
814 | 76 | return MockMetadataClient | 115 | if key in self.data: |
815 | 77 | 116 | r = self.data[key] | |
816 | 78 | 117 | if strip: | |
817 | 79 | class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): | 118 | r = r.strip() |
818 | 119 | else: | ||
819 | 120 | r = default | ||
820 | 121 | return r | ||
821 | 122 | |||
822 | 123 | def get_json(self, key, default=None): | ||
823 | 124 | result = self.get(key, default=default) | ||
824 | 125 | if result is None: | ||
825 | 126 | return default | ||
826 | 127 | return json.loads(result) | ||
827 | 128 | |||
828 | 129 | def exists(self): | ||
829 | 130 | return True | ||
830 | 131 | |||
831 | 132 | |||
832 | 133 | class TestSmartOSDataSource(FilesystemMockingTestCase): | ||
833 | 80 | def setUp(self): | 134 | def setUp(self): |
834 | 81 | super(TestSmartOSDataSource, self).setUp() | 135 | super(TestSmartOSDataSource, self).setUp() |
835 | 82 | 136 | ||
836 | 137 | dsmos = 'cloudinit.sources.DataSourceSmartOS' | ||
837 | 138 | patcher = mock.patch(dsmos + ".jmc_client_factory") | ||
838 | 139 | self.jmc_cfact = patcher.start() | ||
839 | 140 | self.addCleanup(patcher.stop) | ||
840 | 141 | patcher = mock.patch(dsmos + ".get_smartos_environ") | ||
841 | 142 | self.get_smartos_environ = patcher.start() | ||
842 | 143 | self.addCleanup(patcher.stop) | ||
843 | 144 | |||
844 | 83 | self.tmp = tempfile.mkdtemp() | 145 | self.tmp = tempfile.mkdtemp() |
845 | 84 | self.addCleanup(shutil.rmtree, self.tmp) | 146 | self.addCleanup(shutil.rmtree, self.tmp) |
846 | 147 | self.paths = c_helpers.Paths({'cloud_dir': self.tmp}) | ||
847 | 148 | |||
848 | 85 | self.legacy_user_d = tempfile.mkdtemp() | 149 | self.legacy_user_d = tempfile.mkdtemp() |
861 | 86 | self.addCleanup(shutil.rmtree, self.legacy_user_d) | 150 | self.orig_lud = DataSourceSmartOS.LEGACY_USER_D |
862 | 87 | 151 | DataSourceSmartOS.LEGACY_USER_D = self.legacy_user_d | |
851 | 88 | # If you should want to watch the logs... | ||
852 | 89 | self._log = None | ||
853 | 90 | self._log_file = None | ||
854 | 91 | self._log_handler = None | ||
855 | 92 | |||
856 | 93 | # patch cloud_dir, so our 'seed_dir' is guaranteed empty | ||
857 | 94 | self.paths = c_helpers.Paths({'cloud_dir': self.tmp}) | ||
858 | 95 | |||
859 | 96 | self.unapply = [] | ||
860 | 97 | super(TestSmartOSDataSource, self).setUp() | ||
863 | 98 | 152 | ||
864 | 99 | def tearDown(self): | 153 | def tearDown(self): |
869 | 100 | helpers.FilesystemMockingTestCase.tearDown(self) | 154 | DataSourceSmartOS.LEGACY_USER_D = self.orig_lud |
866 | 101 | if self._log_handler and self._log: | ||
867 | 102 | self._log.removeHandler(self._log_handler) | ||
868 | 103 | apply_patches([i for i in reversed(self.unapply)]) | ||
870 | 104 | super(TestSmartOSDataSource, self).tearDown() | 155 | super(TestSmartOSDataSource, self).tearDown() |
871 | 105 | 156 | ||
902 | 106 | def _patchIn(self, root): | 157 | def _get_ds(self, mockdata=None, mode=DataSourceSmartOS.SMARTOS_ENV_KVM, |
903 | 107 | self.restore() | 158 | sys_cfg=None, ds_cfg=None): |
904 | 108 | self.patchOS(root) | 159 | self.jmc_cfact.return_value = PsuedoJoyentClient(mockdata) |
905 | 109 | self.patchUtils(root) | 160 | self.get_smartos_environ.return_value = mode |
876 | 110 | |||
877 | 111 | def apply_patches(self, patches): | ||
878 | 112 | ret = apply_patches(patches) | ||
879 | 113 | self.unapply += ret | ||
880 | 114 | |||
881 | 115 | def _get_ds(self, sys_cfg=None, ds_cfg=None, mockdata=None, dmi_data=None, | ||
882 | 116 | is_lxbrand=False): | ||
883 | 117 | mod = DataSourceSmartOS | ||
884 | 118 | |||
885 | 119 | if mockdata is None: | ||
886 | 120 | mockdata = MOCK_RETURNS | ||
887 | 121 | |||
888 | 122 | if dmi_data is None: | ||
889 | 123 | dmi_data = DMI_DATA_RETURN | ||
890 | 124 | |||
891 | 125 | def _dmi_data(): | ||
892 | 126 | return dmi_data | ||
893 | 127 | |||
894 | 128 | def _os_uname(): | ||
895 | 129 | if not is_lxbrand: | ||
896 | 130 | # LP: #1243287. tests assume this runs, but running test on | ||
897 | 131 | # arm would cause them all to fail. | ||
898 | 132 | return ('LINUX', 'NODENAME', 'RELEASE', 'VERSION', 'x86_64') | ||
899 | 133 | else: | ||
900 | 134 | return ('LINUX', 'NODENAME', 'RELEASE', 'BRANDZ VIRTUAL LINUX', | ||
901 | 135 | 'X86_64') | ||
906 | 136 | 161 | ||
907 | 137 | if sys_cfg is None: | 162 | if sys_cfg is None: |
908 | 138 | sys_cfg = {} | 163 | sys_cfg = {} |
909 | @@ -141,44 +166,8 @@ | |||
910 | 141 | sys_cfg['datasource'] = sys_cfg.get('datasource', {}) | 166 | sys_cfg['datasource'] = sys_cfg.get('datasource', {}) |
911 | 142 | sys_cfg['datasource']['SmartOS'] = ds_cfg | 167 | sys_cfg['datasource']['SmartOS'] = ds_cfg |
912 | 143 | 168 | ||
951 | 144 | self.apply_patches([(mod, 'LEGACY_USER_D', self.legacy_user_d)]) | 169 | return DataSourceSmartOS.DataSourceSmartOS( |
952 | 145 | self.apply_patches([ | 170 | sys_cfg, distro=None, paths=self.paths) |
915 | 146 | (mod, 'JoyentMetadataClient', get_mock_client(mockdata))]) | ||
916 | 147 | self.apply_patches([(mod, 'dmi_data', _dmi_data)]) | ||
917 | 148 | self.apply_patches([(os, 'uname', _os_uname)]) | ||
918 | 149 | self.apply_patches([(mod, 'device_exists', lambda d: True)]) | ||
919 | 150 | dsrc = mod.DataSourceSmartOS(sys_cfg, distro=None, | ||
920 | 151 | paths=self.paths) | ||
921 | 152 | self.apply_patches([(dsrc, '_get_seed_file_object', mock.MagicMock())]) | ||
922 | 153 | return dsrc | ||
923 | 154 | |||
924 | 155 | def test_seed(self): | ||
925 | 156 | # default seed should be /dev/ttyS1 | ||
926 | 157 | dsrc = self._get_ds() | ||
927 | 158 | ret = dsrc.get_data() | ||
928 | 159 | self.assertTrue(ret) | ||
929 | 160 | self.assertEqual('kvm', dsrc.smartos_type) | ||
930 | 161 | self.assertEqual('/dev/ttyS1', dsrc.seed) | ||
931 | 162 | |||
932 | 163 | def test_seed_lxbrand(self): | ||
933 | 164 | # default seed should be /dev/ttyS1 | ||
934 | 165 | dsrc = self._get_ds(is_lxbrand=True) | ||
935 | 166 | ret = dsrc.get_data() | ||
936 | 167 | self.assertTrue(ret) | ||
937 | 168 | self.assertEqual('lx-brand', dsrc.smartos_type) | ||
938 | 169 | self.assertEqual('/native/.zonecontrol/metadata.sock', dsrc.seed) | ||
939 | 170 | |||
940 | 171 | def test_issmartdc(self): | ||
941 | 172 | dsrc = self._get_ds() | ||
942 | 173 | ret = dsrc.get_data() | ||
943 | 174 | self.assertTrue(ret) | ||
944 | 175 | self.assertTrue(dsrc.is_smartdc) | ||
945 | 176 | |||
946 | 177 | def test_issmartdc_lxbrand(self): | ||
947 | 178 | dsrc = self._get_ds(is_lxbrand=True) | ||
948 | 179 | ret = dsrc.get_data() | ||
949 | 180 | self.assertTrue(ret) | ||
950 | 181 | self.assertTrue(dsrc.is_smartdc) | ||
953 | 182 | 171 | ||
954 | 183 | def test_no_base64(self): | 172 | def test_no_base64(self): |
955 | 184 | ds_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True} | 173 | ds_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True} |
956 | @@ -214,58 +203,6 @@ | |||
957 | 214 | self.assertEqual(MOCK_RETURNS['hostname'], | 203 | self.assertEqual(MOCK_RETURNS['hostname'], |
958 | 215 | dsrc.metadata['local-hostname']) | 204 | dsrc.metadata['local-hostname']) |
959 | 216 | 205 | ||
960 | 217 | def test_base64_all(self): | ||
961 | 218 | # metadata provided base64_all of true | ||
962 | 219 | my_returns = MOCK_RETURNS.copy() | ||
963 | 220 | my_returns['base64_all'] = "true" | ||
964 | 221 | for k in ('hostname', 'cloud-init:user-data'): | ||
965 | 222 | my_returns[k] = b64e(my_returns[k]) | ||
966 | 223 | |||
967 | 224 | dsrc = self._get_ds(mockdata=my_returns) | ||
968 | 225 | ret = dsrc.get_data() | ||
969 | 226 | self.assertTrue(ret) | ||
970 | 227 | self.assertEqual(MOCK_RETURNS['hostname'], | ||
971 | 228 | dsrc.metadata['local-hostname']) | ||
972 | 229 | self.assertEqual(MOCK_RETURNS['cloud-init:user-data'], | ||
973 | 230 | dsrc.userdata_raw) | ||
974 | 231 | self.assertEqual(MOCK_RETURNS['root_authorized_keys'], | ||
975 | 232 | dsrc.metadata['public-keys']) | ||
976 | 233 | self.assertEqual(MOCK_RETURNS['disable_iptables_flag'], | ||
977 | 234 | dsrc.metadata['iptables_disable']) | ||
978 | 235 | self.assertEqual(MOCK_RETURNS['enable_motd_sys_info'], | ||
979 | 236 | dsrc.metadata['motd_sys_info']) | ||
980 | 237 | |||
981 | 238 | def test_b64_userdata(self): | ||
982 | 239 | my_returns = MOCK_RETURNS.copy() | ||
983 | 240 | my_returns['b64-cloud-init:user-data'] = "true" | ||
984 | 241 | my_returns['b64-hostname'] = "true" | ||
985 | 242 | for k in ('hostname', 'cloud-init:user-data'): | ||
986 | 243 | my_returns[k] = b64e(my_returns[k]) | ||
987 | 244 | |||
988 | 245 | dsrc = self._get_ds(mockdata=my_returns) | ||
989 | 246 | ret = dsrc.get_data() | ||
990 | 247 | self.assertTrue(ret) | ||
991 | 248 | self.assertEqual(MOCK_RETURNS['hostname'], | ||
992 | 249 | dsrc.metadata['local-hostname']) | ||
993 | 250 | self.assertEqual(MOCK_RETURNS['cloud-init:user-data'], | ||
994 | 251 | dsrc.userdata_raw) | ||
995 | 252 | self.assertEqual(MOCK_RETURNS['root_authorized_keys'], | ||
996 | 253 | dsrc.metadata['public-keys']) | ||
997 | 254 | |||
998 | 255 | def test_b64_keys(self): | ||
999 | 256 | my_returns = MOCK_RETURNS.copy() | ||
1000 | 257 | my_returns['base64_keys'] = 'hostname,ignored' | ||
1001 | 258 | for k in ('hostname',): | ||
1002 | 259 | my_returns[k] = b64e(my_returns[k]) | ||
1003 | 260 | |||
1004 | 261 | dsrc = self._get_ds(mockdata=my_returns) | ||
1005 | 262 | ret = dsrc.get_data() | ||
1006 | 263 | self.assertTrue(ret) | ||
1007 | 264 | self.assertEqual(MOCK_RETURNS['hostname'], | ||
1008 | 265 | dsrc.metadata['local-hostname']) | ||
1009 | 266 | self.assertEqual(MOCK_RETURNS['cloud-init:user-data'], | ||
1010 | 267 | dsrc.userdata_raw) | ||
1011 | 268 | |||
1012 | 269 | def test_userdata(self): | 206 | def test_userdata(self): |
1013 | 270 | dsrc = self._get_ds(mockdata=MOCK_RETURNS) | 207 | dsrc = self._get_ds(mockdata=MOCK_RETURNS) |
1014 | 271 | ret = dsrc.get_data() | 208 | ret = dsrc.get_data() |
1015 | @@ -275,6 +212,13 @@ | |||
1016 | 275 | self.assertEqual(MOCK_RETURNS['cloud-init:user-data'], | 212 | self.assertEqual(MOCK_RETURNS['cloud-init:user-data'], |
1017 | 276 | dsrc.userdata_raw) | 213 | dsrc.userdata_raw) |
1018 | 277 | 214 | ||
1019 | 215 | def test_sdc_nics(self): | ||
1020 | 216 | dsrc = self._get_ds(mockdata=MOCK_RETURNS) | ||
1021 | 217 | ret = dsrc.get_data() | ||
1022 | 218 | self.assertTrue(ret) | ||
1023 | 219 | self.assertEqual(json.loads(MOCK_RETURNS['sdc:nics']), | ||
1024 | 220 | dsrc.metadata['network-data']) | ||
1025 | 221 | |||
1026 | 278 | def test_sdc_scripts(self): | 222 | def test_sdc_scripts(self): |
1027 | 279 | dsrc = self._get_ds(mockdata=MOCK_RETURNS) | 223 | dsrc = self._get_ds(mockdata=MOCK_RETURNS) |
1028 | 280 | ret = dsrc.get_data() | 224 | ret = dsrc.get_data() |
1029 | @@ -430,18 +374,7 @@ | |||
1030 | 430 | mydscfg['disk_aliases']['FOO']) | 374 | mydscfg['disk_aliases']['FOO']) |
1031 | 431 | 375 | ||
1032 | 432 | 376 | ||
1045 | 433 | def apply_patches(patches): | 377 | class TestJoyentMetadataClient(FilesystemMockingTestCase): |
1034 | 434 | ret = [] | ||
1035 | 435 | for (ref, name, replace) in patches: | ||
1036 | 436 | if replace is None: | ||
1037 | 437 | continue | ||
1038 | 438 | orig = getattr(ref, name) | ||
1039 | 439 | setattr(ref, name, replace) | ||
1040 | 440 | ret.append((ref, name, orig)) | ||
1041 | 441 | return ret | ||
1042 | 442 | |||
1043 | 443 | |||
1044 | 444 | class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase): | ||
1046 | 445 | 378 | ||
1047 | 446 | def setUp(self): | 379 | def setUp(self): |
1048 | 447 | super(TestJoyentMetadataClient, self).setUp() | 380 | super(TestJoyentMetadataClient, self).setUp() |
1049 | @@ -481,7 +414,8 @@ | |||
1050 | 481 | mock.Mock(return_value=self.request_id))) | 414 | mock.Mock(return_value=self.request_id))) |
1051 | 482 | 415 | ||
1052 | 483 | def _get_client(self): | 416 | def _get_client(self): |
1054 | 484 | return DataSourceSmartOS.JoyentMetadataClient(self.serial) | 417 | return DataSourceSmartOS.JoyentMetadataClient( |
1055 | 418 | fp=self.serial, smartos_type=DataSourceSmartOS.SMARTOS_ENV_KVM) | ||
1056 | 485 | 419 | ||
1057 | 486 | def assertEndsWith(self, haystack, prefix): | 420 | def assertEndsWith(self, haystack, prefix): |
1058 | 487 | self.assertTrue(haystack.endswith(prefix), | 421 | self.assertTrue(haystack.endswith(prefix), |
1059 | @@ -495,7 +429,7 @@ | |||
1060 | 495 | 429 | ||
1061 | 496 | def test_get_metadata_writes_a_single_line(self): | 430 | def test_get_metadata_writes_a_single_line(self): |
1062 | 497 | client = self._get_client() | 431 | client = self._get_client() |
1064 | 498 | client.get_metadata('some_key') | 432 | client.get('some_key') |
1065 | 499 | self.assertEqual(1, self.serial.write.call_count) | 433 | self.assertEqual(1, self.serial.write.call_count) |
1066 | 500 | written_line = self.serial.write.call_args[0][0] | 434 | written_line = self.serial.write.call_args[0][0] |
1067 | 501 | print(type(written_line)) | 435 | print(type(written_line)) |
1068 | @@ -505,7 +439,7 @@ | |||
1069 | 505 | 439 | ||
1070 | 506 | def _get_written_line(self, key='some_key'): | 440 | def _get_written_line(self, key='some_key'): |
1071 | 507 | client = self._get_client() | 441 | client = self._get_client() |
1073 | 508 | client.get_metadata(key) | 442 | client.get(key) |
1074 | 509 | return self.serial.write.call_args[0][0] | 443 | return self.serial.write.call_args[0][0] |
1075 | 510 | 444 | ||
1076 | 511 | def test_get_metadata_writes_bytes(self): | 445 | def test_get_metadata_writes_bytes(self): |
1077 | @@ -549,32 +483,32 @@ | |||
1078 | 549 | 483 | ||
1079 | 550 | def test_get_metadata_reads_a_line(self): | 484 | def test_get_metadata_reads_a_line(self): |
1080 | 551 | client = self._get_client() | 485 | client = self._get_client() |
1082 | 552 | client.get_metadata('some_key') | 486 | client.get('some_key') |
1083 | 553 | self.assertEqual(self.metasource_data_len, self.serial.read.call_count) | 487 | self.assertEqual(self.metasource_data_len, self.serial.read.call_count) |
1084 | 554 | 488 | ||
1085 | 555 | def test_get_metadata_returns_valid_value(self): | 489 | def test_get_metadata_returns_valid_value(self): |
1086 | 556 | client = self._get_client() | 490 | client = self._get_client() |
1088 | 557 | value = client.get_metadata('some_key') | 491 | value = client.get('some_key') |
1089 | 558 | self.assertEqual(self.metadata_value, value) | 492 | self.assertEqual(self.metadata_value, value) |
1090 | 559 | 493 | ||
1091 | 560 | def test_get_metadata_throws_exception_for_incorrect_length(self): | 494 | def test_get_metadata_throws_exception_for_incorrect_length(self): |
1092 | 561 | self.response_parts['length'] = 0 | 495 | self.response_parts['length'] = 0 |
1093 | 562 | client = self._get_client() | 496 | client = self._get_client() |
1094 | 563 | self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException, | 497 | self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException, |
1096 | 564 | client.get_metadata, 'some_key') | 498 | client.get, 'some_key') |
1097 | 565 | 499 | ||
1098 | 566 | def test_get_metadata_throws_exception_for_incorrect_crc(self): | 500 | def test_get_metadata_throws_exception_for_incorrect_crc(self): |
1099 | 567 | self.response_parts['crc'] = 'deadbeef' | 501 | self.response_parts['crc'] = 'deadbeef' |
1100 | 568 | client = self._get_client() | 502 | client = self._get_client() |
1101 | 569 | self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException, | 503 | self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException, |
1103 | 570 | client.get_metadata, 'some_key') | 504 | client.get, 'some_key') |
1104 | 571 | 505 | ||
1105 | 572 | def test_get_metadata_throws_exception_for_request_id_mismatch(self): | 506 | def test_get_metadata_throws_exception_for_request_id_mismatch(self): |
1106 | 573 | self.response_parts['request_id'] = 'deadbeef' | 507 | self.response_parts['request_id'] = 'deadbeef' |
1107 | 574 | client = self._get_client() | 508 | client = self._get_client() |
1108 | 575 | client._checksum = lambda _: self.response_parts['crc'] | 509 | client._checksum = lambda _: self.response_parts['crc'] |
1109 | 576 | self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException, | 510 | self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException, |
1111 | 577 | client.get_metadata, 'some_key') | 511 | client.get, 'some_key') |
1112 | 578 | 512 | ||
1113 | 579 | def test_get_metadata_returns_None_if_value_not_found(self): | 513 | def test_get_metadata_returns_None_if_value_not_found(self): |
1114 | 580 | self.response_parts['payload'] = '' | 514 | self.response_parts['payload'] = '' |
1115 | @@ -582,4 +516,24 @@ | |||
1116 | 582 | self.response_parts['length'] = 17 | 516 | self.response_parts['length'] = 17 |
1117 | 583 | client = self._get_client() | 517 | client = self._get_client() |
1118 | 584 | client._checksum = lambda _: self.response_parts['crc'] | 518 | client._checksum = lambda _: self.response_parts['crc'] |
1120 | 585 | self.assertIsNone(client.get_metadata('some_key')) | 519 | self.assertIsNone(client.get('some_key')) |
1121 | 520 | |||
1122 | 521 | |||
1123 | 522 | class TestNetworkConversion(TestCase): | ||
1124 | 523 | |||
1125 | 524 | def test_convert_simple(self): | ||
1126 | 525 | expected = { | ||
1127 | 526 | 'version': 1, | ||
1128 | 527 | 'config': [ | ||
1129 | 528 | {'name': 'net0', 'type': 'physical', | ||
1130 | 529 | 'subnets': [{'type': 'static', 'gateway': '8.12.42.1', | ||
1131 | 530 | 'netmask': '255.255.255.0', | ||
1132 | 531 | 'address': '8.12.42.102/24'}], | ||
1133 | 532 | 'mtu': 1500, 'mac_address': '90:b8:d0:f5:e4:f5'}, | ||
1134 | 533 | {'name': 'net1', 'type': 'physical', | ||
1135 | 534 | 'subnets': [{'type': 'static', 'gateway': '192.168.128.1', | ||
1136 | 535 | 'netmask': '255.255.252.0', | ||
1137 | 536 | 'address': '192.168.128.93/22'}], | ||
1138 | 537 | 'mtu': 8500, 'mac_address': '90:b8:d0:a5:ff:cd'}]} | ||
1139 | 538 | found = DataSourceSmartOS.convert_smartos_network_data(SDC_NICS) | ||
1140 | 539 | self.assertEqual(expected, found) |
note, this does depend on https:/ /code.launchpad .net/~smoser/ cloud-init/ trunk.fix- networking/ in order to have the datasource sanely declare that it is 'local'. Otherwise user-data would be pulled and such at local point (without networking).