1
diff --git a/cloudinit/cmd/devel/net_convert.py b/cloudinit/cmd/devel/net_convert.py
2
index 1ec08a3..271dc5e 100755
3
--- a/cloudinit/cmd/devel/net_convert.py
4
+++ b/cloudinit/cmd/devel/net_convert.py
5
@@ -8,6 +8,7 @@ import sys
6
8
import yaml
8
import yaml
7
9
9
8
10
from cloudinit.sources.helpers import openstack
10
from cloudinit.sources.helpers import openstack
9
11
from cloudinit.sources import DataSourceAzure as azure
10
11
12
11
12
from cloudinit.net import eni, netplan, network_state, sysconfig
13
from cloudinit.net import eni, netplan, network_state, sysconfig
12
13
from cloudinit import log
14
from cloudinit import log
13
@@ -28,7 +29,8 @@ def get_parser(parser=None):
14
28
    parser.add_argument("-p", "--network-data", type=open,
29
    parser.add_argument("-p", "--network-data", type=open,
15
29
                        metavar="PATH", required=True)
30
                        metavar="PATH", required=True)
16
30
    parser.add_argument("-k", "--kind",
31
    parser.add_argument("-k", "--kind",
18
31
                        choices=['eni', 'network_data.json', 'yaml'],
32
                        choices=['eni', 'network_data.json', 'yaml',
19
33
                                 'azure-imds'],
20
32
                        required=True)
34
                        required=True)
21
33
    parser.add_argument("-d", "--directory",
35
    parser.add_argument("-d", "--directory",
22
34
                        metavar="PATH",
36
                        metavar="PATH",
23
@@ -78,10 +80,13 @@ def handle_args(name, args):
24
78
                ["Input YAML",
80
                ["Input YAML",
25
79
                 yaml.dump(pre_ns, default_flow_style=False, indent=4), ""]))
81
                 yaml.dump(pre_ns, default_flow_style=False, indent=4), ""]))
26
80
        ns = network_state.parse_net_config_data(pre_ns)
82
        ns = network_state.parse_net_config_data(pre_ns)
28
81
    else:
83
    elif args.kind == 'network_data.json':
29
82
        pre_ns = openstack.convert_net_json(
84
        pre_ns = openstack.convert_net_json(
30
83
            json.loads(net_data), known_macs=known_macs)
85
            json.loads(net_data), known_macs=known_macs)
31
84
        ns = network_state.parse_net_config_data(pre_ns)
86
        ns = network_state.parse_net_config_data(pre_ns)
32
87
    elif args.kind == 'azure-imds':
33
88
        pre_ns = azure.parse_network_config(json.loads(net_data))
34
89
        ns = network_state.parse_net_config_data(pre_ns)
35
85
90
36
86
    if not ns:
91
    if not ns:
37
87
        raise RuntimeError("No valid network_state object created from"
92
        raise RuntimeError("No valid network_state object created from"
38
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
39
index 7007d9e..d8170c3 100644
40
--- a/cloudinit/sources/DataSourceAzure.py
41
+++ b/cloudinit/sources/DataSourceAzure.py
42
@@ -8,6 +8,7 @@ import base64
43
8
import contextlib
8
import contextlib
44
9
import crypt
9
import crypt
45
10
from functools import partial
10
from functools import partial
46
11
import json
47
11
import os
12
import os
48
12
import os.path
13
import os.path
49
13
import re
14
import re
50
@@ -17,6 +18,7 @@ import xml.etree.ElementTree as ET
51
17
18
52
18
from cloudinit import log as logging
19
from cloudinit import log as logging
53
19
from cloudinit import net
20
from cloudinit import net
54
21
from cloudinit.event import EventType
55
20
from cloudinit.net.dhcp import EphemeralDHCPv4
22
from cloudinit.net.dhcp import EphemeralDHCPv4
56
21
from cloudinit import sources
23
from cloudinit import sources
57
22
from cloudinit.sources.helpers.azure import get_metadata_from_fabric
24
from cloudinit.sources.helpers.azure import get_metadata_from_fabric
58
@@ -49,7 +51,17 @@ DEFAULT_FS = 'ext4'
59
49
AZURE_CHASSIS_ASSET_TAG = '7783-7084-3265-9085-8269-3286-77'
51
AZURE_CHASSIS_ASSET_TAG = '7783-7084-3265-9085-8269-3286-77'
60
50
REPROVISION_MARKER_FILE = "/var/lib/cloud/data/poll_imds"
52
REPROVISION_MARKER_FILE = "/var/lib/cloud/data/poll_imds"
61
51
REPORTED_READY_MARKER_FILE = "/var/lib/cloud/data/reported_ready"
53
REPORTED_READY_MARKER_FILE = "/var/lib/cloud/data/reported_ready"
63
52
IMDS_URL = "http://169.254.169.254/metadata/reprovisiondata"
54
AGENT_SEED_DIR = '/var/lib/waagent'
64
55
IMDS_URL = "http://169.254.169.254/metadata/"
65
56
66
57
# List of static scripts and network config artifacts created by
67
58
# stock ubuntu suported images.
68
59
UBUNTU_EXTENDED_NETWORK_SCRIPTS = [
69
60
    '/etc/netplan/90-azure-hotplug.yaml',
70
61
    '/usr/local/sbin/ephemeral_eth.sh',
71
62
    '/etc/udev/rules.d/10-net-device-added.rules',
72
63
    '/run/network/interfaces.ephemeral.d',
73
64
]
74
53
65
75
54
66
76
55
def find_storvscid_from_sysctl_pnpinfo(sysctl_out, deviceid):
67
def find_storvscid_from_sysctl_pnpinfo(sysctl_out, deviceid):
77
@@ -185,7 +197,7 @@ if util.is_FreeBSD():
78
185
197
79
186
BUILTIN_DS_CONFIG = {
198
BUILTIN_DS_CONFIG = {
80
187
    'agent_command': AGENT_START_BUILTIN,
199
    'agent_command': AGENT_START_BUILTIN,
82
188
    'data_dir': "/var/lib/waagent",
200
    'data_dir': AGENT_SEED_DIR,
83
189
    'set_hostname': True,
201
    'set_hostname': True,
84
190
    'hostname_bounce': {
202
    'hostname_bounce': {
85
191
        'interface': DEFAULT_PRIMARY_NIC,
203
        'interface': DEFAULT_PRIMARY_NIC,
86
@@ -252,6 +264,10 @@ class DataSourceAzure(sources.DataSource):
87
252
264
88
253
    dsname = 'Azure'
265
    dsname = 'Azure'
89
254
    _negotiated = False
266
    _negotiated = False
90
267
    _metadata_imds = sources.UNSET
91
268
92
269
    # Regenerate network config new_instance boot and every boot
93
270
    update_events = {'network': [EventType.BOOT_NEW_INSTANCE, EventType.BOOT]}
94
255
271
95
256
    def __init__(self, sys_cfg, distro, paths):
272
    def __init__(self, sys_cfg, distro, paths):
96
257
        sources.DataSource.__init__(self, sys_cfg, distro, paths)
273
        sources.DataSource.__init__(self, sys_cfg, distro, paths)
97
@@ -336,15 +352,17 @@ class DataSourceAzure(sources.DataSource):
98
336
        metadata['public-keys'] = key_value or pubkeys_from_crt_files(fp_files)
352
        metadata['public-keys'] = key_value or pubkeys_from_crt_files(fp_files)
99
337
        return metadata
353
        return metadata
100
338
354
102
339
    def _get_data(self):
355
    def crawl_metadata(self):
103
356
        """Walk all instance metadata sources returning a dict on success.
104
357
105
358
        @return: A dictionary of any metadata content for this instance.
106
359
        @raise: InvalidMetaDataException when the expected metadata service is
107
360
            unavailable, broken or disabled.
108
361
        """
109
362
        crawled_data = {}
110
340
        # azure removes/ejects the cdrom containing the ovf-env.xml
363
        # azure removes/ejects the cdrom containing the ovf-env.xml
111
341
        # file on reboot.  So, in order to successfully reboot we
364
        # file on reboot.  So, in order to successfully reboot we
112
342
        # need to look in the datadir and consider that valid
365
        # need to look in the datadir and consider that valid
113
343
        asset_tag = util.read_dmi_data('chassis-asset-tag')
114
344
        if asset_tag != AZURE_CHASSIS_ASSET_TAG:
115
345
            LOG.debug("Non-Azure DMI asset tag '%s' discovered.", asset_tag)
116
346
            return False
117
347
118
348
        ddir = self.ds_cfg['data_dir']
366
        ddir = self.ds_cfg['data_dir']
119
349
367
120
350
        candidates = [self.seed_dir]
368
        candidates = [self.seed_dir]
121
@@ -373,41 +391,84 @@ class DataSourceAzure(sources.DataSource):
122
373
            except NonAzureDataSource:
391
            except NonAzureDataSource:
123
374
                continue
392
                continue
124
375
            except BrokenAzureDataSource as exc:
393
            except BrokenAzureDataSource as exc:
126
376
                raise exc
394
                msg = 'BrokenAzureDataSource: %s' % exc
127
395
                raise sources.InvalidMetaDataException(msg)
128
377
            except util.MountFailedError:
396
            except util.MountFailedError:
129
378
                LOG.warning("%s was not mountable", cdev)
397
                LOG.warning("%s was not mountable", cdev)
130
379
                continue
398
                continue
131
380
399
132
381
            if reprovision or self._should_reprovision(ret):
400
            if reprovision or self._should_reprovision(ret):
133
382
                ret = self._reprovision()
401
                ret = self._reprovision()
135
383
            (md, self.userdata_raw, cfg, files) = ret
402
            imds_md = get_metadata_from_imds(
136
403
                self.fallback_interface, retries=3)
137
404
            (md, userdata_raw, cfg, files) = ret
138
384
            self.seed = cdev
405
            self.seed = cdev
141
385
            self.metadata = util.mergemanydict([md, DEFAULT_METADATA])
406
            crawled_data.update({
142
386
            self.cfg = util.mergemanydict([cfg, BUILTIN_CLOUD_CONFIG])
407
                'cfg': cfg,
143
408
                'files': files,
144
409
                'metadata': util.mergemanydict(
145
410
                    [md, {'imds': imds_md}]),
146
411
                'userdata_raw': userdata_raw})
147
387
            found = cdev
412
            found = cdev
148
388
413
149
389
            LOG.debug("found datasource in %s", cdev)
414
            LOG.debug("found datasource in %s", cdev)
150
390
            break
415
            break
151
391
416
152
392
        if not found:
417
        if not found:
154
393
            return False
418
            raise sources.InvalidMetaDataException('No Azure metadata found')
155
394
419
156
395
        if found == ddir:
420
        if found == ddir:
157
396
            LOG.debug("using files cached in %s", ddir)
421
            LOG.debug("using files cached in %s", ddir)
158
397
422
159
398
        seed = _get_random_seed()
423
        seed = _get_random_seed()
160
399
        if seed:
424
        if seed:
162
400
            self.metadata['random_seed'] = seed
425
            crawled_data['metadata']['random_seed'] = seed
163
426
        crawled_data['metadata']['instance-id'] = util.read_dmi_data(
164
427
            'system-uuid')
165
428
        return crawled_data
166
429
167
430
    def _is_platform_viable(self):
168
431
        """Check platform environment to report if this datasource may run."""
169
432
        return _is_platform_viable(self.seed_dir)
170
433
171
434
    def clear_cached_attrs(self, attr_defaults=()):
172
435
        """Reset any cached class attributes to defaults."""
173
436
        super(DataSourceAzure, self).clear_cached_attrs(attr_defaults)
174
437
        self._metadata_imds = sources.UNSET
175
438
176
439
    def _get_data(self):
177
440
        """Crawl and process datasource metadata caching metadata as attrs.
178
441
179
442
        @return: True on success, False on error, invalid or disabled
180
443
            datasource.
181
444
        """
182
445
        if not self._is_platform_viable():
183
446
            return False
184
447
        if self.distro and self.distro.name == 'ubuntu':
185
448
            maybe_remove_ubuntu_network_config_scripts()
186
449
        try:
187
450
            crawled_data = util.log_time(
188
451
                        logfunc=LOG.debug, msg='Crawl of metadata service',
189
452
                        func=self.crawl_metadata)
190
453
        except sources.InvalidMetaDataException as e:
191
454
            LOG.warning('Could not crawl Azure metadata: %s', e)
192
455
            return False
193
456
194
457
        # Process crawled data and augment with various config defaults
195
458
        self.cfg = util.mergemanydict(
196
459
            [crawled_data['cfg'], BUILTIN_CLOUD_CONFIG])
197
460
        self._metadata_imds = crawled_data['metadata']['imds']
198
461
        self.metadata = util.mergemanydict(
199
462
            [crawled_data['metadata'], DEFAULT_METADATA])
200
463
        self.userdata_raw = crawled_data['userdata_raw']
201
401
464
202
402
        user_ds_cfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {})
465
        user_ds_cfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {})
203
403
        self.ds_cfg = util.mergemanydict([user_ds_cfg, self.ds_cfg])
466
        self.ds_cfg = util.mergemanydict([user_ds_cfg, self.ds_cfg])
204
404
467
205
405
        # walinux agent writes files world readable, but expects
468
        # walinux agent writes files world readable, but expects
206
406
        # the directory to be protected.
469
        # the directory to be protected.
211
407
        write_files(ddir, files, dirmode=0o700)
470
        write_files(
212
408
471
            self.ds_cfg['data_dir'], crawled_data['files'], dirmode=0o700)
209
409
        self.metadata['instance-id'] = util.read_dmi_data('system-uuid')
210
410
213
411
        return True
472
        return True
214
412
473
215
413
    def device_name_to_device(self, name):
474
    def device_name_to_device(self, name):
216
@@ -436,7 +497,7 @@ class DataSourceAzure(sources.DataSource):
217
436
    def _poll_imds(self):
497
    def _poll_imds(self):
218
437
        """Poll IMDS for the new provisioning data until we get a valid
498
        """Poll IMDS for the new provisioning data until we get a valid
219
438
        response. Then return the returned JSON object."""
499
        response. Then return the returned JSON object."""
221
439
        url = IMDS_URL + "?api-version=2017-04-02"
500
        url = IMDS_URL + "reprovisiondata?api-version=2017-04-02"
222
440
        headers = {"Metadata": "true"}
501
        headers = {"Metadata": "true"}
223
441
        report_ready = bool(not os.path.isfile(REPORTED_READY_MARKER_FILE))
502
        report_ready = bool(not os.path.isfile(REPORTED_READY_MARKER_FILE))
224
442
        LOG.debug("Start polling IMDS")
503
        LOG.debug("Start polling IMDS")
225
@@ -487,7 +548,7 @@ class DataSourceAzure(sources.DataSource):
226
487
        jump back into the polling loop in order to retrieve the ovf_env."""
548
        jump back into the polling loop in order to retrieve the ovf_env."""
227
488
        if not ret:
549
        if not ret:
228
489
            return False
550
            return False
230
490
        (_md, self.userdata_raw, cfg, _files) = ret
551
        (_md, _userdata_raw, cfg, _files) = ret
231
491
        path = REPROVISION_MARKER_FILE
552
        path = REPROVISION_MARKER_FILE
232
492
        if (cfg.get('PreprovisionedVm') is True or
553
        if (cfg.get('PreprovisionedVm') is True or
233
493
                os.path.isfile(path)):
554
                os.path.isfile(path)):
234
@@ -543,22 +604,15 @@ class DataSourceAzure(sources.DataSource):
235
543
    @property
604
    @property
236
544
    def network_config(self):
605
    def network_config(self):
237
545
        """Generate a network config like net.generate_fallback_network() with
606
        """Generate a network config like net.generate_fallback_network() with
239
546
           the following execptions.
607
           the following exceptions.
240
547
608
241
548
           1. Probe the drivers of the net-devices present and inject them in
609
           1. Probe the drivers of the net-devices present and inject them in
242
549
              the network configuration under params: driver: <driver> value
610
              the network configuration under params: driver: <driver> value
243
550
           2. Generate a fallback network config that does not include any of
611
           2. Generate a fallback network config that does not include any of
244
551
              the blacklisted devices.
612
              the blacklisted devices.
245
552
        """
613
        """
246
553
        blacklist = ['mlx4_core']
247
554
        if not self._network_config:
614
        if not self._network_config:
255
555
            LOG.debug('Azure: generating fallback configuration')
615
            self._network_config = parse_network_config(self._metadata_imds)
249
556
            # generate a network config, blacklist picking any mlx4_core devs
250
557
            netconfig = net.generate_fallback_config(
251
558
                blacklist_drivers=blacklist, config_driver=True)
252
559
253
560
            self._network_config = netconfig
254
561
256
562
        return self._network_config
616
        return self._network_config
257
563
617
258
564
618
259
@@ -1025,6 +1079,162 @@ def load_azure_ds_dir(source_dir):
260
1025
    return (md, ud, cfg, {'ovf-env.xml': contents})
1079
    return (md, ud, cfg, {'ovf-env.xml': contents})
261
1026
1080
262
1027
1081
263
1082
def parse_network_config(imds_metadata):
264
1083
    """Convert imds_metadata dictionary to network v2 configuration.
265
1084
266
1085
    Parses network configuration from imds metadata if present or generate
267
1086
    fallback network config excluding mlx4_core devices.
268
1087
269
1088
    @param: imds_metadata: Dict of content read from IMDS network service.
270
1089
    @return: Dictionary containing network version 2 standard configuration.
271
1090
    """
272
1091
    if imds_metadata != sources.UNSET and imds_metadata:
273
1092
        netconfig = {'version': 2, 'ethernets': {}}
274
1093
        LOG.debug('Azure: generating network configuration from IMDS')
275
1094
        network_metadata = imds_metadata['network']
276
1095
        for idx, intf in enumerate(network_metadata['interface']):
277
1096
            nicname = 'eth{idx}'.format(idx=idx)
278
1097
            dev_config = {}
279
1098
            for addr4 in intf['ipv4']['ipAddress']:
280
1099
                privateIpv4 = addr4['privateIpAddress']
281
1100
                if privateIpv4:
282
1101
                    if dev_config.get('dhcp4', False):
283
1102
                        # Append static address config for nic > 1
284
1103
                        netPrefix = intf['ipv4']['subnet'][0].get(
285
1104
                            'prefix', '24')
286
1105
                        if not dev_config.get('addresses'):
287
1106
                            dev_config['addresses'] = []
288
1107
                        dev_config['addresses'].append(
289
1108
                            '{ip}/{prefix}'.format(
290
1109
                                ip=privateIpv4, prefix=netPrefix))
291
1110
                    else:
292
1111
                        dev_config['dhcp4'] = True
293
1112
            for addr6 in intf['ipv6']['ipAddress']:
294
1113
                privateIpv6 = addr6['privateIpAddress']
295
1114
                if privateIpv6:
296
1115
                    dev_config['dhcp6'] = True
297
1116
                    break
298
1117
            if dev_config:
299
1118
                mac = ':'.join(re.findall(r'..', intf['macAddress']))
300
1119
                dev_config.update(
301
1120
                    {'match': {'macaddress': mac.lower()},
302
1121
                     'set-name': nicname})
303
1122
                netconfig['ethernets'][nicname] = dev_config
304
1123
    else:
305
1124
        blacklist = ['mlx4_core']
306
1125
        LOG.debug('Azure: generating fallback configuration')
307
1126
        # generate a network config, blacklist picking mlx4_core devs
308
1127
        netconfig = net.generate_fallback_config(
309
1128
            blacklist_drivers=blacklist, config_driver=True)
310
1129
    return netconfig
311
1130
312
1131
313
1132
def get_metadata_from_imds(fallback_nic, retries):
314
1133
    """Query Azure's network metadata service, returning a dictionary.
315
1134
316
1135
    If network is not up, setup ephemeral dhcp on fallback_nic to talk to the
317
1136
    IMDS. For more info on IMDS:
318
1137
        https://docs.microsoft.com/en-us/azure/virtual-machines/windows/instance-metadata-service
319
1138
320
1139
    @param fallback_nic: String. The name of the nic which requires active
321
1140
        networ in order to query IMDS.
322
1141
    @param retries: The number of retries of the IMDS_URL.
323
1142
324
1143
    @return: A dict of instance metadata containing compute and network
325
1144
        info.
326
1145
    """
327
1146
    if net.is_up(fallback_nic):
328
1147
        return util.log_time(
329
1148
            logfunc=LOG.debug,
330
1149
            msg='Crawl of Azure Instance Metadata Service (IMDS)',
331
1150
            func=_get_metadata_from_imds, args=(retries,))
332
1151
    else:
333
1152
        with EphemeralDHCPv4(fallback_nic):
334
1153
            return util.log_time(
335
1154
                logfunc=LOG.debug,
336
1155
                msg='Crawl of Azure Instance Metadata Service (IMDS)',
337
1156
                func=_get_metadata_from_imds, args=(retries,))
338
1157
339
1158
340
1159
def _get_metadata_from_imds(retries):
341
1160
342
1161
    def retry_on_url_error(msg, exception):
343
1162
        if isinstance(exception, UrlError) and exception.code == 404:
344
1163
            return True  # Continue retries
345
1164
        return False  # Stop retries on all other exceptions, including 404s
346
1165
347
1166
    url = IMDS_URL + "instance?api-version=2017-12-01"
348
1167
    headers = {"Metadata": "true"}
349
1168
    try:
350
1169
        response = readurl(
351
1170
            url, timeout=1, headers=headers, retries=retries,
352
1171
            exception_cb=retry_on_url_error)
353
1172
    except Exception as e:
354
1173
        LOG.debug('Ignoring IMDS instance metadata: %s', e)
355
1174
        return {}
356
1175
    try:
357
1176
        return util.load_json(str(response))
358
1177
    except json.decoder.JSONDecodeError:
359
1178
        LOG.warning(
360
1179
            'Ignoring non-json IMDS instance metadata: %s', str(response))
361
1180
    return {}
362
1181
363
1182
364
1183
def maybe_remove_ubuntu_network_config_scripts(paths=None):
365
1184
    """Remove Azure-specific ubuntu network config for non-primary nics.
366
1185
367
1186
    @param paths: List of networking scripts or directories to remove when
368
1187
        present.
369
1188
370
1189
    In certain supported ubuntu images, static udev rules or netplan yaml
371
1190
    config is delivered in the base ubuntu image to support dhcp on any
372
1191
    additional interfaces which get attached by a customer at some point
373
1192
    after initial boot. Since the Azure datasource can now regenerate
374
1193
    network configuration as metadata reports these new devices, we no longer
375
1194
    want the udev rules or netplan's 90-azure-hotplug.yaml to configure
376
1195
    networking on eth1 or greater as it might collide with cloud-init's
377
1196
    configuration.
378
1197
379
1198
    Remove the any existing extended network scripts if the datasource is
380
1199
    enabled to write network per-boot.
381
1200
    """
382
1201
    if not paths:
383
1202
        paths = UBUNTU_EXTENDED_NETWORK_SCRIPTS
384
1203
    logged = False
385
1204
    for path in paths:
386
1205
        if os.path.exists(path):
387
1206
            if not logged:
388
1207
                LOG.info(
389
1208
                    'Removing Ubuntu extended network scripts because'
390
1209
                    ' cloud-init updates Azure network configuration on the'
391
1210
                    ' following event: %s.',
392
1211
                    EventType.BOOT)
393
1212
                logged = True
394
1213
            if os.path.isdir(path):
395
1214
                util.del_dir(path)
396
1215
            else:
397
1216
                util.del_file(path)
398
1217
399
1218
400
1219
def _is_platform_viable(seed_dir):
401
1220
    """Check platform environment to report if this datasource may run."""
402
1221
    asset_tag = util.read_dmi_data('chassis-asset-tag')
403
1222
    if asset_tag == AZURE_CHASSIS_ASSET_TAG:
404
1223
        return True
405
1224
    LOG.debug("Non-Azure DMI asset tag '%s' discovered.", asset_tag)
406
1225
    for path in (AGENT_SEED_DIR, seed_dir):
407
1226
        if os.path.exists(os.path.join(path, 'ovf-env.xml')):
408
1227
            return True
409
1228
    if util.which('systemd-detect-virt'):
410
1229
        (virt_type, _err) = util.subp(
411
1230
            ['systemd-detect-virt'], rcs=[0, 1], capture=True)
412
1231
        if virt_type.strip() == 'microsoft':
413
1232
            return True
414
1233
    if util.find_devs_with(criteria='LABEL=rd_rdfe_*'):
415
1234
        return True
416
1235
    return False
417
1236
418
1237
419
1028
class BrokenAzureDataSource(Exception):
1238
class BrokenAzureDataSource(Exception):
420
1029
    pass
1239
    pass
421
1030
1240
422
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
423
index c50750a..06e613f 100644
424
--- a/cloudinit/sources/__init__.py
425
+++ b/cloudinit/sources/__init__.py
426
@@ -103,7 +103,6 @@ class DataSource(object):
427
103
    url_timeout = 10    # timeout for each metadata url read attempt
103
    url_timeout = 10    # timeout for each metadata url read attempt
428
104
    url_retries = 5     # number of times to retry url upon 404
104
    url_retries = 5     # number of times to retry url upon 404
429
105
105
430
106
<<<<<<< cloudinit/sources/__init__.py
431
107
    # The datasource defines a set of supported EventTypes during which
106
    # The datasource defines a set of supported EventTypes during which
432
108
    # the datasource can react to changes in metadata and regenerate
107
    # the datasource can react to changes in metadata and regenerate
433
109
    # network configuration on metadata changes.
108
    # network configuration on metadata changes.
434
@@ -112,16 +111,6 @@ class DataSource(object):
435
112
111
436
113
    # Default: generate network config on new instance id (first boot).
112
    # Default: generate network config on new instance id (first boot).
437
114
    update_events = {'network': set([EventType.BOOT_NEW_INSTANCE])}
113
    update_events = {'network': set([EventType.BOOT_NEW_INSTANCE])}
438
115
=======
439
116
    # The datasource defines a list of supported EventTypes during which
440
117
    # the datasource can react to changes in metadata and regenerate
441
118
    # network configuration on metadata changes.
442
119
    # A datasource which supports writing network config on each system boot
443
120
    # would set update_events = {'network': [EventType.BOOT]}
444
121
445
122
    # Default: generate network config on new instance id (first boot).
446
123
    update_events = {'network': [EventType.BOOT_NEW_INSTANCE]}
447
124
>>>>>>> cloudinit/sources/__init__.py
448
125
114
449
126
    # N-tuple listing default values for any metadata-related class
115
    # N-tuple listing default values for any metadata-related class
450
127
    # attributes cached on an instance by a process_data runs. These attribute
116
    # attributes cached on an instance by a process_data runs. These attribute
451
@@ -486,13 +475,8 @@ class DataSource(object):
452
486
            for update_scope, update_events in self.update_events.items():
475
            for update_scope, update_events in self.update_events.items():
453
487
                if event in update_events:
476
                if event in update_events:
454
488
                    if not supported_events.get(update_scope):
477
                    if not supported_events.get(update_scope):
455
489
<<<<<<< cloudinit/sources/__init__.py
456
490
                        supported_events[update_scope] = set()
478
                        supported_events[update_scope] = set()
457
491
                    supported_events[update_scope].add(event)
479
                    supported_events[update_scope].add(event)
458
492
=======
459
493
                        supported_events[update_scope] = []
460
494
                    supported_events[update_scope].append(event)
461
495
>>>>>>> cloudinit/sources/__init__.py
462
496
        for scope, matched_events in supported_events.items():
480
        for scope, matched_events in supported_events.items():
463
497
            LOG.debug(
481
            LOG.debug(
464
498
                "Update datasource metadata and %s config due to events: %s",
482
                "Update datasource metadata and %s config due to events: %s",
465
@@ -506,11 +490,8 @@ class DataSource(object):
466
506
            result = self.get_data()
490
            result = self.get_data()
467
507
            if result:
491
            if result:
468
508
                return True
492
                return True
469
509
<<<<<<< cloudinit/sources/__init__.py
470
510
        LOG.debug("Datasource %s not updated for events: %s", self,
493
        LOG.debug("Datasource %s not updated for events: %s", self,
471
511
                  ', '.join(source_event_types))
494
                  ', '.join(source_event_types))
472
512
=======
473
513
>>>>>>> cloudinit/sources/__init__.py
474
514
        return False
495
        return False
475
515
496
476
516
    def check_instance_id(self, sys_cfg):
497
    def check_instance_id(self, sys_cfg):
477
diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py
478
index 30573cc..9e939c1 100644
479
--- a/cloudinit/sources/tests/test_init.py
480
+++ b/cloudinit/sources/tests/test_init.py
481
@@ -429,14 +429,9 @@ class TestDataSource(CiTestCase):
482
429
429
483
430
    def test_update_metadata_only_acts_on_supported_update_events(self):
430
    def test_update_metadata_only_acts_on_supported_update_events(self):
484
431
        """update_metadata won't get_data on unsupported update events."""
431
        """update_metadata won't get_data on unsupported update events."""
485
432
<<<<<<< cloudinit/sources/tests/test_init.py
486
433
        self.datasource.update_events['network'].discard(EventType.BOOT)
432
        self.datasource.update_events['network'].discard(EventType.BOOT)
487
434
        self.assertEqual(
433
        self.assertEqual(
488
435
            {'network': set([EventType.BOOT_NEW_INSTANCE])},
434
            {'network': set([EventType.BOOT_NEW_INSTANCE])},
489
436
=======
490
437
        self.assertEqual(
491
438
            {'network': [EventType.BOOT_NEW_INSTANCE]},
492
439
>>>>>>> cloudinit/sources/tests/test_init.py
493
440
            self.datasource.update_events)
435
            self.datasource.update_events)
494
441
436
495
442
        def fake_get_data():
437
        def fake_get_data():
496
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
497
index e82716e..5eae247 100644
498
--- a/tests/unittests/test_datasource/test_azure.py
499
+++ b/tests/unittests/test_datasource/test_azure.py
500
@@ -1,15 +1,21 @@
501
1
# This file is part of cloud-init. See LICENSE file for license information.
1
# This file is part of cloud-init. See LICENSE file for license information.
502
2
2
503
3
from cloudinit import distros
504
3
from cloudinit import helpers
4
from cloudinit import helpers
506
4
from cloudinit.sources import DataSourceAzure as dsaz
5
from cloudinit import url_helper
507
6
from cloudinit.sources import (
508
7
    UNSET, DataSourceAzure as dsaz, InvalidMetaDataException)
509
5
from cloudinit.util import (b64e, decode_binary, load_file, write_file,
8
from cloudinit.util import (b64e, decode_binary, load_file, write_file,
510
6
                            find_freebsd_part, get_path_dev_freebsd,
9
                            find_freebsd_part, get_path_dev_freebsd,
511
7
                            MountFailedError)
10
                            MountFailedError)
512
8
from cloudinit.version import version_string as vs
11
from cloudinit.version import version_string as vs
515
9
from cloudinit.tests.helpers import (CiTestCase, TestCase, populate_dir, mock,
12
from cloudinit.tests.helpers import (
516
10
                                     ExitStack, PY26, SkipTest)
13
    HttprettyTestCase, CiTestCase, populate_dir, mock, wrap_and_call,
517
14
    ExitStack, PY26, SkipTest)
518
11
15
519
12
import crypt
16
import crypt
520
17
import httpretty
521
18
import json
522
13
import os
19
import os
523
14
import stat
20
import stat
524
15
import xml.etree.ElementTree as ET
21
import xml.etree.ElementTree as ET
525
@@ -77,6 +83,106 @@ def construct_valid_ovf_env(data=None, pubkeys=None,
526
77
    return content
83
    return content
527
78
84
528
79
85
529
86
NETWORK_METADATA = {
530
87
    "network": {
531
88
        "interface": [
532
89
            {
533
90
                "macAddress": "000D3A047598",
534
91
                "ipv6": {
535
92
                    "ipAddress": []
536
93
                },
537
94
                "ipv4": {
538
95
                    "subnet": [
539
96
                        {
540
97
                           "prefix": "24",
541
98
                           "address": "10.0.0.0"
542
99
                        }
543
100
                    ],
544
101
                    "ipAddress": [
545
102
                        {
546
103
                           "privateIpAddress": "10.0.0.4",
547
104
                           "publicIpAddress": "104.46.124.81"
548
105
                        }
549
106
                    ]
550
107
                }
551
108
            }
552
109
        ]
553
110
    }
554
111
}
555
112
556
113
557
114
class TestGetMetadataFromIMDS(HttprettyTestCase):
558
115
559
116
    with_logs = True
560
117
561
118
    def setUp(self):
562
119
        super(TestGetMetadataFromIMDS, self).setUp()
563
120
        self.network_md_url = dsaz.IMDS_URL + "instance?api-version=2017-12-01"
564
121
565
122
    @mock.patch('cloudinit.sources.DataSourceAzure.readurl')
566
123
    @mock.patch('cloudinit.sources.DataSourceAzure.EphemeralDHCPv4')
567
124
    @mock.patch('cloudinit.sources.DataSourceAzure.net.is_up')
568
125
    def test_get_metadata_does_not_dhcp_if_network_is_up(
569
126
            self, m_net_is_up, m_dhcp, m_readurl):
570
127
        """Do not perform DHCP setup when nic is already up."""
571
128
        m_net_is_up.return_value = True
572
129
        m_readurl.return_value = url_helper.StringResponse(
573
130
            json.dumps(NETWORK_METADATA).encode('utf-8'))
574
131
        self.assertEqual(
575
132
            NETWORK_METADATA,
576
133
            dsaz.get_metadata_from_imds('eth9', retries=3))
577
134
578
135
        m_net_is_up.assert_called_with('eth9')
579
136
        m_dhcp.assert_not_called()
580
137
        self.assertIn(
581
138
            "Crawl of Azure Instance Metadata Service (IMDS) took",  # log_time
582
139
            self.logs.getvalue())
583
140
584
141
    @mock.patch('cloudinit.sources.DataSourceAzure.readurl')
585
142
    @mock.patch('cloudinit.sources.DataSourceAzure.EphemeralDHCPv4')
586
143
    @mock.patch('cloudinit.sources.DataSourceAzure.net.is_up')
587
144
    def test_get_metadata_performs_dhcp_when_network_is_down(
588
145
            self, m_net_is_up, m_dhcp, m_readurl):
589
146
        """Perform DHCP setup when nic is not up."""
590
147
        m_net_is_up.return_value = False
591
148
        m_readurl.return_value = url_helper.StringResponse(
592
149
            json.dumps(NETWORK_METADATA).encode('utf-8'))
593
150
594
151
        self.assertEqual(
595
152
            NETWORK_METADATA,
596
153
            dsaz.get_metadata_from_imds('eth9', retries=2))
597
154
598
155
        m_net_is_up.assert_called_with('eth9')
599
156
        m_dhcp.assert_called_with('eth9')
600
157
        self.assertIn(
601
158
            "Crawl of Azure Instance Metadata Service (IMDS) took",  # log_time
602
159
            self.logs.getvalue())
603
160
604
161
        m_readurl.assert_called_with(
605
162
            self.network_md_url, exception_cb=mock.ANY,
606
163
            headers={'Metadata': 'true'}, retries=2, timeout=1)
607
164
608
165
    @mock.patch('cloudinit.url_helper.time.sleep')
609
166
    @mock.patch('cloudinit.sources.DataSourceAzure.net.is_up')
610
167
    def test_get_metadata_from_imds_empty_when_no_imds_present(
611
168
            self, m_net_is_up, m_sleep):
612
169
        """Return empty dict when IMDS network metadata is absent."""
613
170
        httpretty.register_uri(
614
171
            httpretty.GET,
615
172
            dsaz.IMDS_URL + 'instance?api-version=2017-12-01',
616
173
            body={}, status=404)
617
174
618
175
        m_net_is_up.return_value = True  # skips dhcp
619
176
620
177
        self.assertEqual({}, dsaz.get_metadata_from_imds('eth9', retries=2))
621
178
622
179
        m_net_is_up.assert_called_with('eth9')
623
180
        self.assertEqual([mock.call(1), mock.call(1)], m_sleep.call_args_list)
624
181
        self.assertIn(
625
182
            "Crawl of Azure Instance Metadata Service (IMDS) took",  # log_time
626
183
            self.logs.getvalue())
627
184
628
185
629
80
class TestAzureDataSource(CiTestCase):
186
class TestAzureDataSource(CiTestCase):
630
81
187
631
82
    with_logs = True
188
    with_logs = True
632
@@ -95,8 +201,19 @@ class TestAzureDataSource(CiTestCase):
633
95
        self.patches = ExitStack()
201
        self.patches = ExitStack()
634
96
        self.addCleanup(self.patches.close)
202
        self.addCleanup(self.patches.close)
635
97
203
638
98
        self.patches.enter_context(mock.patch.object(dsaz, '_get_random_seed'))
204
        self.patches.enter_context(mock.patch.object(
639
99
205
            dsaz, '_get_random_seed', return_value='wild'))
640
206
        self.m_get_metadata_from_imds = self.patches.enter_context(
641
207
            mock.patch.object(
642
208
                dsaz, 'get_metadata_from_imds',
643
209
                mock.MagicMock(return_value=NETWORK_METADATA)))
644
210
        self.m_fallback_nic = self.patches.enter_context(
645
211
            mock.patch('cloudinit.sources.net.find_fallback_nic',
646
212
                       return_value='eth9'))
647
213
        self.m_remove_ubuntu_network_scripts = self.patches.enter_context(
648
214
            mock.patch.object(
649
215
                dsaz, 'maybe_remove_ubuntu_network_config_scripts',
650
216
                mock.MagicMock()))
651
100
        super(TestAzureDataSource, self).setUp()
217
        super(TestAzureDataSource, self).setUp()
652
101
218
653
102
    def apply_patches(self, patches):
219
    def apply_patches(self, patches):
654
@@ -137,7 +254,7 @@ scbus-1 on xpt0 bus 0
655
137
        ])
254
        ])
656
138
        return dsaz
255
        return dsaz
657
139
256
659
140
    def _get_ds(self, data, agent_command=None):
257
    def _get_ds(self, data, agent_command=None, distro=None):
660
141
258
661
142
        def dsdevs():
259
        def dsdevs():
662
143
            return data.get('dsdevs', [])
260
            return data.get('dsdevs', [])
663
@@ -186,8 +303,11 @@ scbus-1 on xpt0 bus 0
664
186
                side_effect=_wait_for_files)),
303
                side_effect=_wait_for_files)),
665
187
        ])
304
        ])
666
188
305
667
306
        if distro is not None:
668
307
            distro_cls = distros.fetch(distro)
669
308
            distro = distro_cls(distro, data.get('sys_cfg', {}), self.paths)
670
189
        dsrc = dsaz.DataSourceAzure(
309
        dsrc = dsaz.DataSourceAzure(
672
190
            data.get('sys_cfg', {}), distro=None, paths=self.paths)
310
            data.get('sys_cfg', {}), distro=distro, paths=self.paths)
673
191
        if agent_command is not None:
311
        if agent_command is not None:
674
192
            dsrc.ds_cfg['agent_command'] = agent_command
312
            dsrc.ds_cfg['agent_command'] = agent_command
675
193
313
676
@@ -260,29 +380,20 @@ fdescfs            /dev/fd          fdescfs rw              0 0
677
260
            res = get_path_dev_freebsd('/etc', mnt_list)
380
            res = get_path_dev_freebsd('/etc', mnt_list)
678
261
            self.assertIsNotNone(res)
381
            self.assertIsNotNone(res)
679
262
382
687
263
    @mock.patch('cloudinit.sources.DataSourceAzure.util.read_dmi_data')
383
    @mock.patch('cloudinit.sources.DataSourceAzure._is_platform_viable')
688
264
    def test_non_azure_dmi_chassis_asset_tag(self, m_read_dmi_data):
384
    def test_call_is_platform_viable_seed(self, m_is_platform_viable):
689
265
        """Report non-azure when DMI's chassis asset tag doesn't match.
385
        """Check seed_dir using _is_platform_viable and return False."""
683
266
684
267
        Return False when the asset tag doesn't match Azure's static
685
268
        AZURE_CHASSIS_ASSET_TAG.
686
269
        """
690
270
        # Return a non-matching asset tag value
386
        # Return a non-matching asset tag value
693
271
        nonazure_tag = dsaz.AZURE_CHASSIS_ASSET_TAG + 'X'
387
        m_is_platform_viable.return_value = False
692
272
        m_read_dmi_data.return_value = nonazure_tag
694
273
        dsrc = dsaz.DataSourceAzure(
388
        dsrc = dsaz.DataSourceAzure(
695
274
            {}, distro=None, paths=self.paths)
389
            {}, distro=None, paths=self.paths)
696
275
        self.assertFalse(dsrc.get_data())
390
        self.assertFalse(dsrc.get_data())
701
276
        self.assertEqual(
391
        m_is_platform_viable.assert_called_with(dsrc.seed_dir)
698
277
            "DEBUG: Non-Azure DMI asset tag '{0}' discovered.\n".format(
699
278
                nonazure_tag),
700
279
            self.logs.getvalue())
702
280
392
703
281
    def test_basic_seed_dir(self):
393
    def test_basic_seed_dir(self):
704
282
        odata = {'HostName': "myhost", 'UserName': "myuser"}
394
        odata = {'HostName': "myhost", 'UserName': "myuser"}
705
283
        data = {'ovfcontent': construct_valid_ovf_env(data=odata),
395
        data = {'ovfcontent': construct_valid_ovf_env(data=odata),
706
284
                'sys_cfg': {}}
396
                'sys_cfg': {}}
707
285
708
286
        dsrc = self._get_ds(data)
397
        dsrc = self._get_ds(data)
709
287
        ret = dsrc.get_data()
398
        ret = dsrc.get_data()
710
288
        self.assertTrue(ret)
399
        self.assertTrue(ret)
711
@@ -291,6 +402,82 @@ fdescfs            /dev/fd          fdescfs rw              0 0
712
291
        self.assertTrue(os.path.isfile(
402
        self.assertTrue(os.path.isfile(
713
292
            os.path.join(self.waagent_d, 'ovf-env.xml')))
403
            os.path.join(self.waagent_d, 'ovf-env.xml')))
714
293
404
715
405
    def test_get_data_non_ubuntu_will_not_remove_network_scripts(self):
716
406
        """get_data on non-Ubuntu will not remove ubuntu net scripts."""
717
407
        odata = {'HostName': "myhost", 'UserName': "myuser"}
718
408
        data = {'ovfcontent': construct_valid_ovf_env(data=odata),
719
409
                'sys_cfg': {}}
720
410
721
411
        dsrc = self._get_ds(data, distro='debian')
722
412
        dsrc.get_data()
723
413
        self.m_remove_ubuntu_network_scripts.assert_not_called()
724
414
725
415
    def test_get_data_on_ubuntu_will_remove_network_scripts(self):
726
416
        """get_data will remove ubuntu net scripts on Ubuntu distro."""
727
417
        odata = {'HostName': "myhost", 'UserName': "myuser"}
728
418
        data = {'ovfcontent': construct_valid_ovf_env(data=odata),
729
419
                'sys_cfg': {}}
730
420
731
421
        dsrc = self._get_ds(data, distro='ubuntu')
732
422
        dsrc.get_data()
733
423
        self.m_remove_ubuntu_network_scripts.assert_called_once_with()
734
424
735
425
    def test_crawl_metadata_returns_structured_data_and_caches_nothing(self):
736
426
        """Return all structured metadata and cache no class attributes."""
737
427
        yaml_cfg = "{agent_command: my_command}\n"
738
428
        odata = {'HostName': "myhost", 'UserName': "myuser",
739
429
                 'UserData': {'text': 'FOOBAR', 'encoding': 'plain'},
740
430
                 'dscfg': {'text': yaml_cfg, 'encoding': 'plain'}}
741
431
        data = {'ovfcontent': construct_valid_ovf_env(data=odata),
742
432
                'sys_cfg': {}}
743
433
        dsrc = self._get_ds(data)
744
434
        expected_cfg = {
745
435
            'PreprovisionedVm': False,
746
436
            'datasource': {'Azure': {'agent_command': 'my_command'}},
747
437
            'system_info': {'default_user': {'name': u'myuser'}}}
748
438
        expected_metadata = {
749
439
            'azure_data': {
750
440
                'configurationsettype': 'LinuxProvisioningConfiguration'},
751
441
            'imds': {'network': {'interface': [{
752
442
                'ipv4': {'ipAddress': [
753
443
                     {'privateIpAddress': '10.0.0.4',
754
444
                      'publicIpAddress': '104.46.124.81'}],
755
445
                      'subnet': [{'address': '10.0.0.0', 'prefix': '24'}]},
756
446
                'ipv6': {'ipAddress': []},
757
447
                'macAddress': '000D3A047598'}]}},
758
448
            'instance-id': 'test-instance-id',
759
449
            'local-hostname': u'myhost',
760
450
            'random_seed': 'wild'}
761
451
762
452
        crawled_metadata = dsrc.crawl_metadata()
763
453
764
454
        self.assertItemsEqual(
765
455
            crawled_metadata.keys(),
766
456
            ['cfg', 'files', 'metadata', 'userdata_raw'])
767
457
        self.assertEqual(crawled_metadata['cfg'], expected_cfg)
768
458
        self.assertEqual(
769
459
            list(crawled_metadata['files'].keys()), ['ovf-env.xml'])
770
460
        self.assertIn(
771
461
            b'<HostName>myhost</HostName>',
772
462
            crawled_metadata['files']['ovf-env.xml'])
773
463
        self.assertEqual(crawled_metadata['metadata'], expected_metadata)
774
464
        self.assertEqual(crawled_metadata['userdata_raw'], 'FOOBAR')
775
465
        self.assertEqual(dsrc.userdata_raw, None)
776
466
        self.assertEqual(dsrc.metadata, {})
777
467
        self.assertEqual(dsrc._metadata_imds, UNSET)
778
468
        self.assertFalse(os.path.isfile(
779
469
            os.path.join(self.waagent_d, 'ovf-env.xml')))
780
470
781
471
    def test_crawl_metadata_raises_invalid_metadata_on_error(self):
782
472
        """crawl_metadata raises an exception on invalid ovf-env.xml."""
783
473
        data = {'ovfcontent': "BOGUS", 'sys_cfg': {}}
784
474
        dsrc = self._get_ds(data)
785
475
        error_msg = ('BrokenAzureDataSource: Invalid ovf-env.xml:'
786
476
                     ' syntax error: line 1, column 0')
787
477
        with self.assertRaises(InvalidMetaDataException) as cm:
788
478
            dsrc.crawl_metadata()
789
479
        self.assertEqual(str(cm.exception), error_msg)
790
480
791
294
    def test_waagent_d_has_0700_perms(self):
481
    def test_waagent_d_has_0700_perms(self):
792
295
        # we expect /var/lib/waagent to be created 0700
482
        # we expect /var/lib/waagent to be created 0700
793
296
        dsrc = self._get_ds({'ovfcontent': construct_valid_ovf_env()})
483
        dsrc = self._get_ds({'ovfcontent': construct_valid_ovf_env()})
794
@@ -314,6 +501,20 @@ fdescfs            /dev/fd          fdescfs rw              0 0
795
314
        self.assertTrue(ret)
501
        self.assertTrue(ret)
796
315
        self.assertEqual(data['agent_invoked'], cfg['agent_command'])
502
        self.assertEqual(data['agent_invoked'], cfg['agent_command'])
797
316
503
798
504
    def test_network_config_set_from_imds(self):
799
505
        """Datasource.network_config returns IMDS network data."""
800
506
        odata = {}
801
507
        data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
802
508
        expected_network_config = {
803
509
            'ethernets': {
804
510
                'eth0': {'set-name': 'eth0',
805
511
                         'match': {'macaddress': '00:0d:3a:04:75:98'},
806
512
                         'dhcp4': True}},
807
513
            'version': 2}
808
514
        dsrc = self._get_ds(data)
809
515
        dsrc.get_data()
810
516
        self.assertEqual(expected_network_config, dsrc.network_config)
811
517
812
317
    def test_user_cfg_set_agent_command(self):
518
    def test_user_cfg_set_agent_command(self):
813
318
        # set dscfg in via base64 encoded yaml
519
        # set dscfg in via base64 encoded yaml
814
319
        cfg = {'agent_command': "my_command"}
520
        cfg = {'agent_command': "my_command"}
815
@@ -579,12 +780,34 @@ fdescfs            /dev/fd          fdescfs rw              0 0
816
579
        self.assertEqual(
780
        self.assertEqual(
817
580
            [mock.call("/dev/cd0")], m_check_fbsd_cdrom.call_args_list)
781
            [mock.call("/dev/cd0")], m_check_fbsd_cdrom.call_args_list)
818
581
782
819
783
    @mock.patch('cloudinit.net.generate_fallback_config')
820
784
    def test_imds_network_config(self, mock_fallback):
821
785
        """Network config is generated from IMDS network data when present."""
822
786
        odata = {'HostName': "myhost", 'UserName': "myuser"}
823
787
        data = {'ovfcontent': construct_valid_ovf_env(data=odata),
824
788
                'sys_cfg': {}}
825
789
826
790
        dsrc = self._get_ds(data)
827
791
        ret = dsrc.get_data()
828
792
        self.assertTrue(ret)
829
793
830
794
        expected_cfg = {
831
795
            'ethernets': {
832
796
                'eth0': {'dhcp4': True,
833
797
                         'match': {'macaddress': '00:0d:3a:04:75:98'},
834
798
                         'set-name': 'eth0'}},
835
799
            'version': 2}
836
800
837
801
        self.assertEqual(expected_cfg, dsrc.network_config)
838
802
        mock_fallback.assert_not_called()
839
803
840
582
    @mock.patch('cloudinit.net.get_interface_mac')
804
    @mock.patch('cloudinit.net.get_interface_mac')
841
583
    @mock.patch('cloudinit.net.get_devicelist')
805
    @mock.patch('cloudinit.net.get_devicelist')
842
584
    @mock.patch('cloudinit.net.device_driver')
806
    @mock.patch('cloudinit.net.device_driver')
843
585
    @mock.patch('cloudinit.net.generate_fallback_config')
807
    @mock.patch('cloudinit.net.generate_fallback_config')
846
586
    def test_network_config(self, mock_fallback, mock_dd,
808
    def test_fallback_network_config(self, mock_fallback, mock_dd,
847
587
                            mock_devlist, mock_get_mac):
809
                                     mock_devlist, mock_get_mac):
848
810
        """On absent IMDS network data, generate network fallback config."""
849
588
        odata = {'HostName': "myhost", 'UserName': "myuser"}
811
        odata = {'HostName': "myhost", 'UserName': "myuser"}
850
589
        data = {'ovfcontent': construct_valid_ovf_env(data=odata),
812
        data = {'ovfcontent': construct_valid_ovf_env(data=odata),
851
590
                'sys_cfg': {}}
813
                'sys_cfg': {}}
852
@@ -605,6 +828,8 @@ fdescfs            /dev/fd          fdescfs rw              0 0
853
605
        mock_get_mac.return_value = '00:11:22:33:44:55'
828
        mock_get_mac.return_value = '00:11:22:33:44:55'
854
606
829
855
607
        dsrc = self._get_ds(data)
830
        dsrc = self._get_ds(data)
856
831
        # Represent empty response from network imds
857
832
        self.m_get_metadata_from_imds.return_value = {}
858
608
        ret = dsrc.get_data()
833
        ret = dsrc.get_data()
859
609
        self.assertTrue(ret)
834
        self.assertTrue(ret)
860
610
835
861
@@ -617,8 +842,9 @@ fdescfs            /dev/fd          fdescfs rw              0 0
862
617
    @mock.patch('cloudinit.net.get_devicelist')
842
    @mock.patch('cloudinit.net.get_devicelist')
863
618
    @mock.patch('cloudinit.net.device_driver')
843
    @mock.patch('cloudinit.net.device_driver')
864
619
    @mock.patch('cloudinit.net.generate_fallback_config')
844
    @mock.patch('cloudinit.net.generate_fallback_config')
867
620
    def test_network_config_blacklist(self, mock_fallback, mock_dd,
845
    def test_fallback_network_config_blacklist(self, mock_fallback, mock_dd,
868
621
                                      mock_devlist, mock_get_mac):
846
                                               mock_devlist, mock_get_mac):
869
847
        """On absent network metadata, blacklist mlx from fallback config."""
870
622
        odata = {'HostName': "myhost", 'UserName': "myuser"}
848
        odata = {'HostName': "myhost", 'UserName': "myuser"}
871
623
        data = {'ovfcontent': construct_valid_ovf_env(data=odata),
849
        data = {'ovfcontent': construct_valid_ovf_env(data=odata),
872
624
                'sys_cfg': {}}
850
                'sys_cfg': {}}
873
@@ -649,6 +875,8 @@ fdescfs            /dev/fd          fdescfs rw              0 0
874
649
        mock_get_mac.return_value = '00:11:22:33:44:55'
875
        mock_get_mac.return_value = '00:11:22:33:44:55'
875
650
876
876
651
        dsrc = self._get_ds(data)
877
        dsrc = self._get_ds(data)
877
878
        # Represent empty response from network imds
878
879
        self.m_get_metadata_from_imds.return_value = {}
879
652
        ret = dsrc.get_data()
880
        ret = dsrc.get_data()
880
653
        self.assertTrue(ret)
881
        self.assertTrue(ret)
881
654
882
882
@@ -689,9 +917,12 @@ class TestAzureBounce(CiTestCase):
883
689
            mock.patch.object(dsaz, 'get_metadata_from_fabric',
917
            mock.patch.object(dsaz, 'get_metadata_from_fabric',
884
690
                              mock.MagicMock(return_value={})))
918
                              mock.MagicMock(return_value={})))
885
691
        self.patches.enter_context(
919
        self.patches.enter_context(
887
692
            mock.patch.object(dsaz.util, 'which', lambda x: True))
920
            mock.patch.object(dsaz, 'get_metadata_from_imds',
888
921
                              mock.MagicMock(return_value={})))
889
693
        self.patches.enter_context(
922
        self.patches.enter_context(
891
694
            mock.patch.object(dsaz, '_get_random_seed'))
923
            mock.patch.object(dsaz.util, 'which', lambda x: True))
892
924
        self.patches.enter_context(mock.patch.object(
893
925
            dsaz, '_get_random_seed', return_value='wild'))
894
695
926
895
696
        def _dmi_mocks(key):
927
        def _dmi_mocks(key):
896
697
            if key == 'system-uuid':
928
            if key == 'system-uuid':
897
@@ -719,9 +950,12 @@ class TestAzureBounce(CiTestCase):
898
719
            mock.patch.object(dsaz, 'set_hostname'))
950
            mock.patch.object(dsaz, 'set_hostname'))
899
720
        self.subp = self.patches.enter_context(
951
        self.subp = self.patches.enter_context(
900
721
            mock.patch('cloudinit.sources.DataSourceAzure.util.subp'))
952
            mock.patch('cloudinit.sources.DataSourceAzure.util.subp'))
901
953
        self.find_fallback_nic = self.patches.enter_context(
902
954
            mock.patch('cloudinit.net.find_fallback_nic', return_value='eth9'))
903
722
955
904
723
    def tearDown(self):
956
    def tearDown(self):
905
724
        self.patches.close()
957
        self.patches.close()
906
958
        super(TestAzureBounce, self).tearDown()
907
725
959
908
726
    def _get_ds(self, ovfcontent=None, agent_command=None):
960
    def _get_ds(self, ovfcontent=None, agent_command=None):
909
727
        if ovfcontent is not None:
961
        if ovfcontent is not None:
910
@@ -927,7 +1161,7 @@ class TestLoadAzureDsDir(CiTestCase):
911
927
            str(context_manager.exception))
1161
            str(context_manager.exception))
912
928
1162
913
929
1163
915
930
class TestReadAzureOvf(TestCase):
1164
class TestReadAzureOvf(CiTestCase):
916
931
1165
917
932
    def test_invalid_xml_raises_non_azure_ds(self):
1166
    def test_invalid_xml_raises_non_azure_ds(self):
918
933
        invalid_xml = "<foo>" + construct_valid_ovf_env(data={})
1167
        invalid_xml = "<foo>" + construct_valid_ovf_env(data={})
919
@@ -1188,6 +1422,25 @@ class TestCanDevBeReformatted(CiTestCase):
920
1188
                      "(datasource.Azure.never_destroy_ntfs)", msg)
1422
                      "(datasource.Azure.never_destroy_ntfs)", msg)
921
1189
1423
922
1190
1424
923
1425
class TestClearCachedData(CiTestCase):
924
1426
925
1427
    def test_clear_cached_attrs_clears_imds(self):
926
1428
        """All class attributes are reset to defaults, including imds data."""
927
1429
        tmp = self.tmp_dir()
928
1430
        paths = helpers.Paths(
929
1431
            {'cloud_dir': tmp, 'run_dir': tmp})
930
1432
        dsrc = dsaz.DataSourceAzure({}, distro=None, paths=paths)
931
1433
        clean_values = [dsrc.metadata, dsrc.userdata, dsrc._metadata_imds]
932
1434
        dsrc.metadata = 'md'
933
1435
        dsrc.userdata = 'ud'
934
1436
        dsrc._metadata_imds = 'imds'
935
1437
        dsrc._dirty_cache = True
936
1438
        dsrc.clear_cached_attrs()
937
1439
        self.assertEqual(
938
1440
            [dsrc.metadata, dsrc.userdata, dsrc._metadata_imds],
939
1441
            clean_values)
940
1442
941
1443
942
1191
class TestAzureNetExists(CiTestCase):
1444
class TestAzureNetExists(CiTestCase):
943
1192
1445
944
1193
    def test_azure_net_must_exist_for_legacy_objpkl(self):
1446
    def test_azure_net_must_exist_for_legacy_objpkl(self):
945
@@ -1398,4 +1651,128 @@ class TestAzureDataSourcePreprovisioning(CiTestCase):
946
1398
        self.assertEqual(m_net.call_count, 1)
1651
        self.assertEqual(m_net.call_count, 1)
947
1399
1652
948
1400
1653
949
1654
class TestRemoveUbuntuNetworkConfigScripts(CiTestCase):
950
1655
951
1656
    with_logs = True
952
1657
953
1658
    def setUp(self):
954
1659
        super(TestRemoveUbuntuNetworkConfigScripts, self).setUp()
955
1660
        self.tmp = self.tmp_dir()
956
1661
957
1662
    def test_remove_network_scripts_removes_both_files_and_directories(self):
958
1663
        """Any files or directories in paths are removed when present."""
959
1664
        file1 = self.tmp_path('file1', dir=self.tmp)
960
1665
        subdir = self.tmp_path('sub1', dir=self.tmp)
961
1666
        subfile = self.tmp_path('leaf1', dir=subdir)
962
1667
        write_file(file1, 'file1content')
963
1668
        write_file(subfile, 'leafcontent')
964
1669
        dsaz.maybe_remove_ubuntu_network_config_scripts(paths=[subdir, file1])
965
1670
966
1671
        for path in (file1, subdir, subfile):
967
1672
            self.assertFalse(os.path.exists(path),
968
1673
                             'Found unremoved: %s' % path)
969
1674
970
1675
        expected_logs = [
971
1676
            'INFO: Removing Ubuntu extended network scripts because cloud-init'
972
1677
            ' updates Azure network configuration on the following event:'
973
1678
            ' System boot.',
974
1679
            'Recursively deleting %s' % subdir,
975
1680
            'Attempting to remove %s' % file1]
976
1681
        for log in expected_logs:
977
1682
            self.assertIn(log, self.logs.getvalue())
978
1683
979
1684
    def test_remove_network_scripts_only_attempts_removal_if_path_exists(self):
980
1685
        """Any files or directories absent are skipped without error."""
981
1686
        dsaz.maybe_remove_ubuntu_network_config_scripts(paths=[
982
1687
            self.tmp_path('nodirhere/', dir=self.tmp),
983
1688
            self.tmp_path('notfilehere', dir=self.tmp)])
984
1689
        self.assertNotIn('/not/a', self.logs.getvalue())  # No delete logs
985
1690
986
1691
    @mock.patch('cloudinit.sources.DataSourceAzure.os.path.exists')
987
1692
    def test_remove_network_scripts_default_removes_stock_scripts(self,
988
1693
                                                                  m_exists):
989
1694
        """Azure's stock ubuntu image scripts and artifacts are removed."""
990
1695
        # Report path absent on all to avoid delete operation
991
1696
        m_exists.return_value = False
992
1697
        dsaz.maybe_remove_ubuntu_network_config_scripts()
993
1698
        calls = m_exists.call_args_list
994
1699
        for path in dsaz.UBUNTU_EXTENDED_NETWORK_SCRIPTS:
995
1700
            self.assertIn(mock.call(path), calls)
996
1701
997
1702
998
1703
class TestWBIsPlatformViable(CiTestCase):
999
1704
    """White box tests for _is_platform_viable."""
1000
1705
    with_logs = True
1001
1706
1002
1707
    @mock.patch('cloudinit.sources.DataSourceAzure.util.read_dmi_data')
1003
1708
    def test_true_on_non_azure_chassis(self, m_read_dmi_data):
1004
1709
        """Return True if DMI chassis-asset-tag is AZURE_CHASSIS_ASSET_TAG."""
1005
1710
        m_read_dmi_data.return_value = dsaz.AZURE_CHASSIS_ASSET_TAG
1006
1711
        self.assertTrue(dsaz._is_platform_viable('doesnotmatter'))
1007
1712
1008
1713
    @mock.patch('cloudinit.sources.DataSourceAzure.os.path.exists')
1009
1714
    @mock.patch('cloudinit.sources.DataSourceAzure.util.read_dmi_data')
1010
1715
    def test_true_on_azure_ovf_env_in_seed_dir(self, m_read_dmi_data, m_exist):
1011
1716
        """Return True if ovf-env.xml exists in known seed dirs."""
1012
1717
        # Non-matching Azure chassis-asset-tag
1013
1718
        m_read_dmi_data.return_value = dsaz.AZURE_CHASSIS_ASSET_TAG + 'X'
1014
1719
1015
1720
        m_exist.side_effect = [False, True]
1016
1721
        self.assertTrue(dsaz._is_platform_viable('/other/seed/dir'))
1017
1722
        self.assertEqual(
1018
1723
            [mock.call('/var/lib/waagent/ovf-env.xml'),
1019
1724
             mock.call('/other/seed/dir/ovf-env.xml')],
1020
1725
            m_exist.call_args_list)
1021
1726
1022
1727
    @mock.patch('cloudinit.sources.DataSourceAzure.util.which')
1023
1728
    @mock.patch('cloudinit.sources.DataSourceAzure.util.subp')
1024
1729
    def test_true_on_detect_virt_microsoft(self, m_subp, m_which):
1025
1730
        """Return True if a partition label prefix rd_rdfe is present."""
1026
1731
        m_which.return_value = '/usr/bin/systemd-detect-virt'
1027
1732
        m_subp.return_value = ('microsoft\n', '')
1028
1733
        self.assertTrue(wrap_and_call(
1029
1734
            'cloudinit.sources.DataSourceAzure',
1030
1735
            {'os.path.exists': False,
1031
1736
             # Non-matching Azure chassis-asset-tag
1032
1737
             'util.read_dmi_data': dsaz.AZURE_CHASSIS_ASSET_TAG + 'X'},
1033
1738
            dsaz._is_platform_viable, 'doesnotmatter'))
1034
1739
        m_which.assert_called_once_with('systemd-detect-virt')
1035
1740
        m_subp.assert_called_once_with(
1036
1741
            ['systemd-detect-virt'], capture=True, rcs=[0, 1])
1037
1742
1038
1743
    @mock.patch('cloudinit.sources.DataSourceAzure.util.find_devs_with')
1039
1744
    def test_true_on_azure_when_fs_label_prefix_rd_rdfe(self, m_find_devs):
1040
1745
        """Return True if a partition label prefix rd_rdfe is present."""
1041
1746
1042
1747
        m_find_devs.return_value = ['/dev/Imatched/azure']
1043
1748
        self.assertTrue(wrap_and_call(
1044
1749
            'cloudinit.sources.DataSourceAzure',
1045
1750
            {'os.path.exists': False,
1046
1751
             # Non-matching Azure chassis-asset-tag
1047
1752
             'util.read_dmi_data': dsaz.AZURE_CHASSIS_ASSET_TAG + 'X',
1048
1753
             'util.which': None},
1049
1754
            dsaz._is_platform_viable, 'doesnotmatter'))
1050
1755
        m_find_devs.assert_called_once_with(criteria='LABEL=rd_rdfe_*')
1051
1756
1052
1757
    def test_false_on_no_matching_azure_criteria(self):
1053
1758
        """Report non-azure on unmatched asset tag, ovf-env absent and no dev.
1054
1759
1055
1760
        Return False when the asset tag doesn't match Azure's static
1056
1761
        AZURE_CHASSIS_ASSET_TAG, no ovf-env.xml files exist in known seed dirs
1057
1762
        and no devices have a label starting with prefix 'rd_rdfe_'.
1058
1763
        """
1059
1764
        self.assertFalse(wrap_and_call(
1060
1765
            'cloudinit.sources.DataSourceAzure',
1061
1766
            {'os.path.exists': False,
1062
1767
             # Non-matching Azure chassis-asset-tag
1063
1768
             'util.read_dmi_data': dsaz.AZURE_CHASSIS_ASSET_TAG + 'X',
1064
1769
             'util.which': None,
1065
1770
             'util.find_devs_with': []},
1066
1771
            dsaz._is_platform_viable, 'doesnotmatter'))
1067
1772
        self.assertIn(
1068
1773
            "DEBUG: Non-Azure DMI asset tag '{0}' discovered.\n".format(
1069
1774
                dsaz.AZURE_CHASSIS_ASSET_TAG + 'X'),
1070
1775
            self.logs.getvalue())
1071
1776
1072
1777
1073
1401
# vi: ts=4 expandtab
1778
# vi: ts=4 expandtab
Status:	Superseded
Proposed branch:	~chad.smith/cloud-init:feature/azure-network-on-boot
Merge into:	cloud-init:master
Prerequisite:	~chad.smith/cloud-init:feature/maintain-network-on-boot
Diff against target:	1073 lines (+651/-83) 5 files modified cloudinit/cmd/devel/net_convert.py (+7/-2) cloudinit/sources/DataSourceAzure.py (+239/-29) cloudinit/sources/__init__.py (+0/-19) cloudinit/sources/tests/test_init.py (+0/-5) tests/unittests/test_datasource/test_azure.py (+405/-28)
Related bugs:	Link a bug report
Reviewer	Review Type	Date Requested	Status
Server Team CI bot	continuous-integration		Approve on 2018-08-07
cloud-init Commiters		2018-06-28	Pending
Review via email: mp+348704@code.launchpad.net