1
diff --git a/.gitignore b/.gitignore
2
index b0500a6..75565ed 100644
3
--- a/.gitignore
4
+++ b/.gitignore
5
@@ -10,3 +10,4 @@ parts
6
10
prime
10
prime
7
11
stage
11
stage
8
12
*.snap
12
*.snap
9
13
*.cover
10
diff --git a/cloudinit/net/dhcp.py b/cloudinit/net/dhcp.py
11
index 875a460..087c0c0 100644
12
--- a/cloudinit/net/dhcp.py
13
+++ b/cloudinit/net/dhcp.py
14
@@ -10,7 +10,9 @@ import os
15
10
import re
10
import re
16
11
import signal
11
import signal
17
12
12
19
13
from cloudinit.net import find_fallback_nic, get_devicelist
13
from cloudinit.net import (
20
14
    EphemeralIPv4Network, find_fallback_nic, get_devicelist)
21
15
from cloudinit.net.network_state import mask_and_ipv4_to_bcast_addr as bcip
22
14
from cloudinit import temp_utils
16
from cloudinit import temp_utils
23
15
from cloudinit import util
17
from cloudinit import util
24
16
from six import StringIO
18
from six import StringIO
25
@@ -29,6 +31,45 @@ class InvalidDHCPLeaseFileError(Exception):
26
29
    pass
31
    pass
27
30
32
28
31
33
29
34
class NoDHCPLeaseError(Exception):
30
35
    """Raised when unable to get a DHCP lease."""
31
36
    pass
32
37
33
38
34
39
class EphemeralDHCPv4(object):
35
40
    def __init__(self, iface=None):
36
41
        self.iface = iface
37
42
        self._ephipv4 = None
38
43
39
44
    def __enter__(self):
40
45
        try:
41
46
            leases = maybe_perform_dhcp_discovery(self.iface)
42
47
        except InvalidDHCPLeaseFileError:
43
48
            raise NoDHCPLeaseError()
44
49
        if not leases:
45
50
            raise NoDHCPLeaseError()
46
51
        lease = leases[-1]
47
52
        LOG.debug("Received dhcp lease on %s for %s/%s",
48
53
                  lease['interface'], lease['fixed-address'],
49
54
                  lease['subnet-mask'])
50
55
        nmap = {'interface': 'interface', 'ip': 'fixed-address',
51
56
                'prefix_or_mask': 'subnet-mask',
52
57
                'broadcast': 'broadcast-address',
53
58
                'router': 'routers'}
54
59
        kwargs = dict([(k, lease.get(v)) for k, v in nmap.items()])
55
60
        if not kwargs['broadcast']:
56
61
            kwargs['broadcast'] = bcip(kwargs['prefix_or_mask'], kwargs['ip'])
57
62
        ephipv4 = EphemeralIPv4Network(**kwargs)
58
63
        ephipv4.__enter__()
59
64
        self._ephipv4 = ephipv4
60
65
        return lease
61
66
62
67
    def __exit__(self, excp_type, excp_value, excp_traceback):
63
68
        if not self._ephipv4:
64
69
            return
65
70
        self._ephipv4.__exit__(excp_type, excp_value, excp_traceback)
66
71
67
72
68
32
def maybe_perform_dhcp_discovery(nic=None):
73
def maybe_perform_dhcp_discovery(nic=None):
69
33
    """Perform dhcp discovery if nic valid and dhclient command exists.
74
    """Perform dhcp discovery if nic valid and dhclient command exists.
70
34
75
71
diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py
72
index 31738c7..fe667d8 100644
73
--- a/cloudinit/net/network_state.py
74
+++ b/cloudinit/net/network_state.py
75
@@ -961,4 +961,16 @@ def mask_to_net_prefix(mask):
76
961
        return ipv4_mask_to_net_prefix(mask)
961
        return ipv4_mask_to_net_prefix(mask)
77
962
962
78
963
963
79
964
def mask_and_ipv4_to_bcast_addr(mask, ip):
80
965
    """Calculate the broadcast address from the subnet mask and ip addr.
81
966
82
967
    Supports ipv4 only."""
83
968
    ip_bin = int(''.join([bin(int(x) + 256)[3:] for x in ip.split('.')]), 2)
84
969
    mask_dec = ipv4_mask_to_net_prefix(mask)
85
970
    bcast_bin = ip_bin | (2**(32 - mask_dec) - 1)
86
971
    bcast_str = '.'.join([str(bcast_bin >> (i << 3) & 0xFF)
87
972
                          for i in range(4)[::-1]])
88
973
    return bcast_str
89
974
90
975
91
964
# vi: ts=4 expandtab
976
# vi: ts=4 expandtab
92
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
93
index d1d0975..4bcbf3a 100644
94
--- a/cloudinit/sources/DataSourceAzure.py
95
+++ b/cloudinit/sources/DataSourceAzure.py
96
@@ -11,13 +11,16 @@ from functools import partial
97
11
import os
11
import os
98
12
import os.path
12
import os.path
99
13
import re
13
import re
100
14
from time import time
101
14
from xml.dom import minidom
15
from xml.dom import minidom
102
15
import xml.etree.ElementTree as ET
16
import xml.etree.ElementTree as ET
103
16
17
104
17
from cloudinit import log as logging
18
from cloudinit import log as logging
105
18
from cloudinit import net
19
from cloudinit import net
106
20
from cloudinit.net.dhcp import EphemeralDHCPv4
107
19
from cloudinit import sources
21
from cloudinit import sources
108
20
from cloudinit.sources.helpers.azure import get_metadata_from_fabric
22
from cloudinit.sources.helpers.azure import get_metadata_from_fabric
109
23
from cloudinit.url_helper import readurl, wait_for_url, UrlError
110
21
from cloudinit import util
24
from cloudinit import util
111
22
25
112
23
LOG = logging.getLogger(__name__)
26
LOG = logging.getLogger(__name__)
113
@@ -44,6 +47,9 @@ LEASE_FILE = '/var/lib/dhcp/dhclient.eth0.leases'
114
44
DEFAULT_FS = 'ext4'
47
DEFAULT_FS = 'ext4'
115
45
# DMI chassis-asset-tag is set static for all azure instances
48
# DMI chassis-asset-tag is set static for all azure instances
116
46
AZURE_CHASSIS_ASSET_TAG = '7783-7084-3265-9085-8269-3286-77'
49
AZURE_CHASSIS_ASSET_TAG = '7783-7084-3265-9085-8269-3286-77'
117
50
REPROVISION_MARKER_FILE = "/var/lib/cloud/data/poll_imds"
118
51
IMDS_URL = "http://169.254.169.254/metadata/reprovisiondata"
119
52
IMDS_RETRIES = 5
120
47
53
121
48
54
122
49
def find_storvscid_from_sysctl_pnpinfo(sysctl_out, deviceid):
55
def find_storvscid_from_sysctl_pnpinfo(sysctl_out, deviceid):
123
@@ -276,19 +282,20 @@ class DataSourceAzure(sources.DataSource):
124
276
282
125
277
        with temporary_hostname(azure_hostname, self.ds_cfg,
283
        with temporary_hostname(azure_hostname, self.ds_cfg,
126
278
                                hostname_command=hostname_command) \
284
                                hostname_command=hostname_command) \
129
279
                as previous_hostname:
285
                as previous_hn:
130
280
            if (previous_hostname is not None and
286
            if (previous_hn is not None and
131
281
                    util.is_true(self.ds_cfg.get('set_hostname'))):
287
                    util.is_true(self.ds_cfg.get('set_hostname'))):
132
282
                cfg = self.ds_cfg['hostname_bounce']
288
                cfg = self.ds_cfg['hostname_bounce']
133
283
289
134
284
                # "Bouncing" the network
290
                # "Bouncing" the network
135
285
                try:
291
                try:
139
286
                    perform_hostname_bounce(hostname=azure_hostname,
292
                    return perform_hostname_bounce(hostname=azure_hostname,
140
287
                                            cfg=cfg,
293
                                                   cfg=cfg,
141
288
                                            prev_hostname=previous_hostname)
294
                                                   prev_hostname=previous_hn)
142
289
                except Exception as e:
295
                except Exception as e:
143
290
                    LOG.warning("Failed publishing hostname: %s", e)
296
                    LOG.warning("Failed publishing hostname: %s", e)
144
291
                    util.logexc(LOG, "handling set_hostname failed")
297
                    util.logexc(LOG, "handling set_hostname failed")
145
298
        return False
146
292
299
147
293
    def get_metadata_from_agent(self):
300
    def get_metadata_from_agent(self):
148
294
        temp_hostname = self.metadata.get('local-hostname')
301
        temp_hostname = self.metadata.get('local-hostname')
149
@@ -345,15 +352,20 @@ class DataSourceAzure(sources.DataSource):
150
345
        ddir = self.ds_cfg['data_dir']
352
        ddir = self.ds_cfg['data_dir']
151
346
353
152
347
        candidates = [self.seed_dir]
354
        candidates = [self.seed_dir]
153
355
        if os.path.isfile(REPROVISION_MARKER_FILE):
154
356
            candidates.insert(0, "IMDS")
155
348
        candidates.extend(list_possible_azure_ds_devs())
357
        candidates.extend(list_possible_azure_ds_devs())
156
349
        if ddir:
358
        if ddir:
157
350
            candidates.append(ddir)
359
            candidates.append(ddir)
158
351
360
159
352
        found = None
361
        found = None
161
353
362
        reprovision = False
162
354
        for cdev in candidates:
363
        for cdev in candidates:
163
355
            try:
364
            try:
165
356
                if cdev.startswith("/dev/"):
365
                if cdev == "IMDS":
166
366
                    ret = None
167
367
                    reprovision = True
168
368
                elif cdev.startswith("/dev/"):
169
357
                    if util.is_FreeBSD():
369
                    if util.is_FreeBSD():
170
358
                        ret = util.mount_cb(cdev, load_azure_ds_dir,
370
                        ret = util.mount_cb(cdev, load_azure_ds_dir,
171
359
                                            mtype="udf", sync=False)
371
                                            mtype="udf", sync=False)
172
@@ -370,6 +382,8 @@ class DataSourceAzure(sources.DataSource):
173
370
                LOG.warning("%s was not mountable", cdev)
382
                LOG.warning("%s was not mountable", cdev)
174
371
                continue
383
                continue
175
372
384
176
385
            if reprovision or self._should_reprovision(ret):
177
386
                ret = self._reprovision()
178
373
            (md, self.userdata_raw, cfg, files) = ret
387
            (md, self.userdata_raw, cfg, files) = ret
179
374
            self.seed = cdev
388
            self.seed = cdev
180
375
            self.metadata = util.mergemanydict([md, DEFAULT_METADATA])
389
            self.metadata = util.mergemanydict([md, DEFAULT_METADATA])
181
@@ -428,6 +442,83 @@ class DataSourceAzure(sources.DataSource):
182
428
            LOG.debug("negotiating already done for %s",
442
            LOG.debug("negotiating already done for %s",
183
429
                      self.get_instance_id())
443
                      self.get_instance_id())
184
430
444
185
445
    def _poll_imds(self, report_ready=True):
186
446
        """Poll IMDS for the new provisioning data until we get a valid
187
447
        response. Then return the returned JSON object."""
188
448
        url = IMDS_URL + "?api-version=2017-04-02"
189
449
        headers = {"Metadata": "true"}
190
450
        LOG.debug("Start polling IMDS")
191
451
192
452
        def sleep_cb(response, loop_n):
193
453
            return 1
194
454
195
455
        def exception_cb(msg, exception):
196
456
            if isinstance(exception, UrlError) and exception.code == 404:
197
457
                return
198
458
            LOG.warning("Exception during polling. Will try DHCP.",
199
459
                        exc_info=True)
200
460
201
461
            # If we get an exception while trying to call IMDS, we
202
462
            # call DHCP and setup the ephemeral network to acquire the new IP.
203
463
            raise exception
204
464
205
465
        need_report = report_ready
206
466
        for i in range(IMDS_RETRIES):
207
467
            try:
208
468
                with EphemeralDHCPv4() as lease:
209
469
                    if need_report:
210
470
                        self._report_ready(lease=lease)
211
471
                        need_report = False
212
472
                    wait_for_url([url], max_wait=None, timeout=60,
213
473
                                 status_cb=LOG.info,
214
474
                                 headers_cb=lambda url: headers, sleep_time=1,
215
475
                                 exception_cb=exception_cb,
216
476
                                 sleep_time_cb=sleep_cb)
217
477
                    return str(readurl(url, headers=headers))
218
478
            except Exception:
219
479
                LOG.debug("Exception during polling-retrying dhcp" +
220
480
                          " %d more time(s).", (IMDS_RETRIES - i),
221
481
                          exc_info=True)
222
482
223
483
    def _report_ready(self, lease):
224
484
        """Tells the fabric provisioning has completed
225
485
           before we go into our polling loop."""
226
486
        try:
227
487
            get_metadata_from_fabric(None, lease['unknown-245'])
228
488
        except Exception as exc:
229
489
            LOG.warning(
230
490
                "Error communicating with Azure fabric; You may experience."
231
491
                "connectivity issues.", exc_info=True)
232
492
233
493
    def _should_reprovision(self, ret):
234
494
        """Whether or not we should poll IMDS for reprovisioning data.
235
495
        Also sets a marker file to poll IMDS.
236
496
237
497
        The marker file is used for the following scenario: the VM boots into
238
498
        this polling loop, which we expect to be proceeding infinitely until
239
499
        the VM is picked. If for whatever reason the platform moves us to a
240
500
        new host (for instance a hardware issue), we need to keep polling.
241
501
        However, since the VM reports ready to the Fabric, we will not attach
242
502
        the ISO, thus cloud-init needs to have a way of knowing that it should
243
503
        jump back into the polling loop in order to retrieve the ovf_env."""
244
504
        if not ret:
245
505
            return False
246
506
        (md, self.userdata_raw, cfg, files) = ret
247
507
        path = REPROVISION_MARKER_FILE
248
508
        if (cfg.get('PreprovisionedVm') is True or
249
509
                os.path.isfile(path)):
250
510
            if not os.path.isfile(path):
251
511
                LOG.info("Creating a marker file to poll imds")
252
512
                util.write_file(path, "%s: %s\n" % (os.getpid(), time()))
253
513
            return True
254
514
        return False
255
515
256
516
    def _reprovision(self):
257
517
        """Initiate the reprovisioning workflow."""
258
518
        contents = self._poll_imds()
259
519
        md, ud, cfg = read_azure_ovf(contents)
260
520
        return (md, ud, cfg, {'ovf-env.xml': contents})
261
521
262
431
    def _negotiate(self):
522
    def _negotiate(self):
263
432
        """Negotiate with fabric and return data from it.
523
        """Negotiate with fabric and return data from it.
264
433
524
265
@@ -453,7 +544,7 @@ class DataSourceAzure(sources.DataSource):
266
453
                "Error communicating with Azure fabric; You may experience."
544
                "Error communicating with Azure fabric; You may experience."
267
454
                "connectivity issues.", exc_info=True)
545
                "connectivity issues.", exc_info=True)
268
455
            return False
546
            return False
270
456
547
        util.del_file(REPROVISION_MARKER_FILE)
271
457
        return fabric_data
548
        return fabric_data
272
458
549
273
459
    def activate(self, cfg, is_new_instance):
550
    def activate(self, cfg, is_new_instance):
274
@@ -595,6 +686,7 @@ def address_ephemeral_resize(devpath=RESOURCE_DISK_PATH, maxwait=120,
275
595
def perform_hostname_bounce(hostname, cfg, prev_hostname):
686
def perform_hostname_bounce(hostname, cfg, prev_hostname):
276
596
    # set the hostname to 'hostname' if it is not already set to that.
687
    # set the hostname to 'hostname' if it is not already set to that.
277
597
    # then, if policy is not off, bounce the interface using command
688
    # then, if policy is not off, bounce the interface using command
278
689
    # Returns True if the network was bounced, False otherwise.
279
598
    command = cfg['command']
690
    command = cfg['command']
280
599
    interface = cfg['interface']
691
    interface = cfg['interface']
281
600
    policy = cfg['policy']
692
    policy = cfg['policy']
282
@@ -614,7 +706,8 @@ def perform_hostname_bounce(hostname, cfg, prev_hostname):
283
614
        else:
706
        else:
284
615
            LOG.debug(
707
            LOG.debug(
285
616
                "Skipping network bounce: ifupdown utils aren't present.")
708
                "Skipping network bounce: ifupdown utils aren't present.")
287
617
            return  # Don't bounce as networkd handles hostname DDNS updates
709
            # Don't bounce as networkd handles hostname DDNS updates
288
710
            return False
289
618
    LOG.debug("pubhname: publishing hostname [%s]", msg)
711
    LOG.debug("pubhname: publishing hostname [%s]", msg)
290
619
    shell = not isinstance(command, (list, tuple))
712
    shell = not isinstance(command, (list, tuple))
291
620
    # capture=False, see comments in bug 1202758 and bug 1206164.
713
    # capture=False, see comments in bug 1202758 and bug 1206164.
292
@@ -622,6 +715,7 @@ def perform_hostname_bounce(hostname, cfg, prev_hostname):
293
622
                  get_uptime=True, func=util.subp,
715
                  get_uptime=True, func=util.subp,
294
623
                  kwargs={'args': command, 'shell': shell, 'capture': False,
716
                  kwargs={'args': command, 'shell': shell, 'capture': False,
295
624
                          'env': env})
717
                          'env': env})
296
718
    return True
297
625
719
298
626
720
299
627
def crtfile_to_pubkey(fname, data=None):
721
def crtfile_to_pubkey(fname, data=None):
300
@@ -838,9 +932,35 @@ def read_azure_ovf(contents):
301
838
    if 'ssh_pwauth' not in cfg and password:
932
    if 'ssh_pwauth' not in cfg and password:
302
839
        cfg['ssh_pwauth'] = True
933
        cfg['ssh_pwauth'] = True
303
840
934
304
935
    cfg['PreprovisionedVm'] = _extract_preprovisioned_vm_setting(dom)
305
936
306
841
    return (md, ud, cfg)
937
    return (md, ud, cfg)
307
842
938
308
843
939
309
940
def _extract_preprovisioned_vm_setting(dom):
310
941
    """Read the preprovision flag from the ovf. It should not
311
942
       exist unless true."""
312
943
    platform_settings_section = find_child(
313
944
        dom.documentElement,
314
945
        lambda n: n.localName == "PlatformSettingsSection")
315
946
    if not platform_settings_section or len(platform_settings_section) == 0:
316
947
        LOG.debug("PlatformSettingsSection not found")
317
948
        return False
318
949
    platform_settings = find_child(
319
950
        platform_settings_section[0],
320
951
        lambda n: n.localName == "PlatformSettings")
321
952
    if not platform_settings or len(platform_settings) == 0:
322
953
        LOG.debug("PlatformSettings not found")
323
954
        return False
324
955
    preprovisionedVm = find_child(
325
956
        platform_settings[0],
326
957
        lambda n: n.localName == "PreprovisionedVm")
327
958
    if not preprovisionedVm or len(preprovisionedVm) == 0:
328
959
        LOG.debug("PreprovisionedVm not found")
329
960
        return False
330
961
    return util.translate_bool(preprovisionedVm[0].firstChild.nodeValue)
331
962
332
963
333
844
def encrypt_pass(password, salt_id="$6$"):
964
def encrypt_pass(password, salt_id="$6$"):
334
845
    return crypt.crypt(password, salt_id + util.rand_str(strlen=16))
965
    return crypt.crypt(password, salt_id + util.rand_str(strlen=16))
335
846
966
336
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
337
index 0f89f34..e14553b 100644
338
--- a/cloudinit/sources/DataSourceEc2.py
339
+++ b/cloudinit/sources/DataSourceEc2.py
340
@@ -14,7 +14,7 @@ import time
341
14
from cloudinit import ec2_utils as ec2
14
from cloudinit import ec2_utils as ec2
342
15
from cloudinit import log as logging
15
from cloudinit import log as logging
343
16
from cloudinit import net
16
from cloudinit import net
345
17
from cloudinit.net import dhcp
17
from cloudinit.net.dhcp import EphemeralDHCPv4, NoDHCPLeaseError
346
18
from cloudinit import sources
18
from cloudinit import sources
347
19
from cloudinit import url_helper as uhelp
19
from cloudinit import url_helper as uhelp
348
20
from cloudinit import util
20
from cloudinit import util
349
@@ -102,22 +102,13 @@ class DataSourceEc2(sources.DataSource):
350
102
            if util.is_FreeBSD():
102
            if util.is_FreeBSD():
351
103
                LOG.debug("FreeBSD doesn't support running dhclient with -sf")
103
                LOG.debug("FreeBSD doesn't support running dhclient with -sf")
352
104
                return False
104
                return False
358
105
            dhcp_leases = dhcp.maybe_perform_dhcp_discovery(
105
            try:
359
106
                self.fallback_interface)
106
                with EphemeralDHCPv4(self.fallback_interface):
360
107
            if not dhcp_leases:
107
                    return util.log_time(
361
108
                # DataSourceEc2Local failed in init-local stage. DataSourceEc2
108
                        logfunc=LOG.debug, msg='Crawl of metadata service',
362
109
                # will still run in init-network stage.
109
                        func=self._crawl_metadata)
363
110
            except NoDHCPLeaseError:
364
110
                return False
111
                return False
365
111
            dhcp_opts = dhcp_leases[-1]
366
112
            net_params = {'interface': dhcp_opts.get('interface'),
367
113
                          'ip': dhcp_opts.get('fixed-address'),
368
114
                          'prefix_or_mask': dhcp_opts.get('subnet-mask'),
369
115
                          'broadcast': dhcp_opts.get('broadcast-address'),
370
116
                          'router': dhcp_opts.get('routers')}
371
117
            with net.EphemeralIPv4Network(**net_params):
372
118
                return util.log_time(
373
119
                    logfunc=LOG.debug, msg='Crawl of metadata service',
374
120
                    func=self._crawl_metadata)
375
121
        else:
112
        else:
376
122
            return self._crawl_metadata()
113
            return self._crawl_metadata()
377
123
114
378
diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py
379
index 6cda572..90c12df 100644
380
--- a/cloudinit/sources/helpers/azure.py
381
+++ b/cloudinit/sources/helpers/azure.py
382
@@ -199,10 +199,10 @@ class WALinuxAgentShim(object):
383
199
        '  </Container>',
199
        '  </Container>',
384
200
        '</Health>'])
200
        '</Health>'])
385
201
201
387
202
    def __init__(self, fallback_lease_file=None):
202
    def __init__(self, fallback_lease_file=None, dhcp_options=None):
388
203
        LOG.debug('WALinuxAgentShim instantiated, fallback_lease_file=%s',
203
        LOG.debug('WALinuxAgentShim instantiated, fallback_lease_file=%s',
389
204
                  fallback_lease_file)
204
                  fallback_lease_file)
391
205
        self.dhcpoptions = None
205
        self.dhcpoptions = dhcp_options
392
206
        self._endpoint = None
206
        self._endpoint = None
393
207
        self.openssl_manager = None
207
        self.openssl_manager = None
394
208
        self.values = {}
208
        self.values = {}
395
@@ -220,7 +220,8 @@ class WALinuxAgentShim(object):
396
220
    @property
220
    @property
397
221
    def endpoint(self):
221
    def endpoint(self):
398
222
        if self._endpoint is None:
222
        if self._endpoint is None:
400
223
            self._endpoint = self.find_endpoint(self.lease_file)
223
            self._endpoint = self.find_endpoint(self.lease_file,
401
224
                                                self.dhcpoptions)
402
224
        return self._endpoint
225
        return self._endpoint
403
225
226
404
226
    @staticmethod
227
    @staticmethod
405
@@ -292,10 +293,14 @@ class WALinuxAgentShim(object):
406
292
        return _value
293
        return _value
407
293
294
408
294
    @staticmethod
295
    @staticmethod
410
295
    def find_endpoint(fallback_lease_file=None):
296
    def find_endpoint(fallback_lease_file=None, dhcp245=None):
411
296
        value = None
297
        value = None
414
297
        LOG.debug('Finding Azure endpoint from networkd...')
298
        if dhcp245 is not None:
415
298
        value = WALinuxAgentShim._networkd_get_value_from_leases()
299
            value = dhcp245
416
300
            LOG.debug("Using Azure Endpoint from dhcp options")
417
301
        if value is None:
418
302
            LOG.debug('Finding Azure endpoint from networkd...')
419
303
            value = WALinuxAgentShim._networkd_get_value_from_leases()
420
299
        if value is None:
304
        if value is None:
421
300
            # Option-245 stored in /run/cloud-init/dhclient.hooks/<ifc>.json
305
            # Option-245 stored in /run/cloud-init/dhclient.hooks/<ifc>.json
422
301
            # a dhclient exit hook that calls cloud-init-dhclient-hook
306
            # a dhclient exit hook that calls cloud-init-dhclient-hook
423
@@ -367,8 +372,9 @@ class WALinuxAgentShim(object):
424
367
        LOG.info('Reported ready to Azure fabric.')
372
        LOG.info('Reported ready to Azure fabric.')
425
368
373
426
369
374
429
370
def get_metadata_from_fabric(fallback_lease_file=None):
375
def get_metadata_from_fabric(fallback_lease_file=None, dhcp_opts=None):
430
371
    shim = WALinuxAgentShim(fallback_lease_file=fallback_lease_file)
376
    shim = WALinuxAgentShim(fallback_lease_file=fallback_lease_file,
431
377
                            dhcp_options=dhcp_opts)
432
372
    try:
378
    try:
433
373
        return shim.register_with_azure_and_fetch_data()
379
        return shim.register_with_azure_and_fetch_data()
434
374
    finally:
380
    finally:
435
diff --git a/cloudinit/temp_utils.py b/cloudinit/temp_utils.py
436
index 5d7adf7..c98a1b5 100644
437
--- a/cloudinit/temp_utils.py
438
+++ b/cloudinit/temp_utils.py
439
@@ -28,13 +28,18 @@ def _tempfile_dir_arg(odir=None, needs_exe=False):
440
28
    if odir is not None:
28
    if odir is not None:
441
29
        return odir
29
        return odir
442
30
30
443
31
    if needs_exe:
444
32
        tdir = _EXE_ROOT_TMPDIR
445
33
        if not os.path.isdir(tdir):
446
34
            os.makedirs(tdir)
447
35
            os.chmod(tdir, 0o1777)
448
36
        return tdir
449
37
450
31
    global _TMPDIR
38
    global _TMPDIR
451
32
    if _TMPDIR:
39
    if _TMPDIR:
452
33
        return _TMPDIR
40
        return _TMPDIR
453
34
41
457
35
    if needs_exe:
42
    if os.getuid() == 0:
455
36
        tdir = _EXE_ROOT_TMPDIR
456
37
    elif os.getuid() == 0:
458
38
        tdir = _ROOT_TMPDIR
43
        tdir = _ROOT_TMPDIR
459
39
    else:
44
    else:
460
40
        tdir = os.environ.get('TMPDIR', '/tmp')
45
        tdir = os.environ.get('TMPDIR', '/tmp')
461
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
462
index 0e0f5b4..0a5be0b 100644
463
--- a/cloudinit/url_helper.py
464
+++ b/cloudinit/url_helper.py
465
@@ -273,7 +273,7 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
466
273
273
467
274
def wait_for_url(urls, max_wait=None, timeout=None,
274
def wait_for_url(urls, max_wait=None, timeout=None,
468
275
                 status_cb=None, headers_cb=None, sleep_time=1,
275
                 status_cb=None, headers_cb=None, sleep_time=1,
470
276
                 exception_cb=None):
276
                 exception_cb=None, sleep_time_cb=None):
471
277
    """
277
    """
472
278
    urls:      a list of urls to try
278
    urls:      a list of urls to try
473
279
    max_wait:  roughly the maximum time to wait before giving up
279
    max_wait:  roughly the maximum time to wait before giving up
474
@@ -286,6 +286,8 @@ def wait_for_url(urls, max_wait=None, timeout=None,
475
286
                for request.
286
                for request.
476
287
    exception_cb: call method with 2 arguments 'msg' (per status_cb) and
287
    exception_cb: call method with 2 arguments 'msg' (per status_cb) and
477
288
                  'exception', the exception that occurred.
288
                  'exception', the exception that occurred.
478
289
    sleep_time_cb: call method with 2 arguments (response, loop_n) that
479
290
                   generates the next sleep time.
480
289
291
481
290
    the idea of this routine is to wait for the EC2 metdata service to
292
    the idea of this routine is to wait for the EC2 metdata service to
482
291
    come up.  On both Eucalyptus and EC2 we have seen the case where
293
    come up.  On both Eucalyptus and EC2 we have seen the case where
483
@@ -301,6 +303,8 @@ def wait_for_url(urls, max_wait=None, timeout=None,
484
301
    service but is not going to find one.  It is possible that the instance
303
    service but is not going to find one.  It is possible that the instance
485
302
    data host (169.254.169.254) may be firewalled off Entirely for a sytem,
304
    data host (169.254.169.254) may be firewalled off Entirely for a sytem,
486
303
    meaning that the connection will block forever unless a timeout is set.
305
    meaning that the connection will block forever unless a timeout is set.
487
306
488
307
    A value of None for max_wait will retry indefinitely.
489
304
    """
308
    """
490
305
    start_time = time.time()
309
    start_time = time.time()
491
306
310
492
@@ -311,18 +315,24 @@ def wait_for_url(urls, max_wait=None, timeout=None,
493
311
        status_cb = log_status_cb
315
        status_cb = log_status_cb
494
312
316
495
313
    def timeup(max_wait, start_time):
317
    def timeup(max_wait, start_time):
498
314
        return ((max_wait <= 0 or max_wait is None) or
318
        if (max_wait is None):
499
315
                (time.time() - start_time > max_wait))
319
            return False
500
320
        return ((max_wait <= 0) or (time.time() - start_time > max_wait))
501
316
321
502
317
    loop_n = 0
322
    loop_n = 0
503
323
    response = None
504
318
    while True:
324
    while True:
506
319
        sleep_time = int(loop_n / 5) + 1
325
        if sleep_time_cb is not None:
507
326
            sleep_time = sleep_time_cb(response, loop_n)
508
327
        else:
509
328
            sleep_time = int(loop_n / 5) + 1
510
320
        for url in urls:
329
        for url in urls:
511
321
            now = time.time()
330
            now = time.time()
512
322
            if loop_n != 0:
331
            if loop_n != 0:
513
323
                if timeup(max_wait, start_time):
332
                if timeup(max_wait, start_time):
514
324
                    break
333
                    break
516
325
                if timeout and (now + timeout > (start_time + max_wait)):
334
                if (max_wait is not None and
517
335
                        timeout and (now + timeout > (start_time + max_wait))):
518
326
                    # shorten timeout to not run way over max_time
336
                    # shorten timeout to not run way over max_time
519
327
                    timeout = int((start_time + max_wait) - now)
337
                    timeout = int((start_time + max_wait) - now)
520
328
338
521
@@ -354,10 +364,11 @@ def wait_for_url(urls, max_wait=None, timeout=None,
522
354
                url_exc = e
364
                url_exc = e
523
355
365
524
356
            time_taken = int(time.time() - start_time)
366
            time_taken = int(time.time() - start_time)
529
357
            status_msg = "Calling '%s' failed [%s/%ss]: %s" % (url,
367
            max_wait_str = "%ss" % max_wait if max_wait else "unlimited"
530
358
                                                               time_taken,
368
            status_msg = "Calling '%s' failed [%s/%s]: %s" % (url,
531
359
                                                               max_wait,
369
                                                              time_taken,
532
360
                                                               reason)
370
                                                              max_wait_str,
533
371
                                                              reason)
534
361
            status_cb(status_msg)
372
            status_cb(status_msg)
535
362
            if exception_cb:
373
            if exception_cb:
536
363
                # This can be used to alter the headers that will be sent
374
                # This can be used to alter the headers that will be sent
537
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
538
index 6341e1e..254e987 100644
539
--- a/tests/unittests/test_datasource/test_azure.py
540
+++ b/tests/unittests/test_datasource/test_azure.py
541
@@ -5,7 +5,7 @@ from cloudinit.util import b64e, decode_binary, load_file, write_file
542
5
from cloudinit.sources import DataSourceAzure as dsaz
5
from cloudinit.sources import DataSourceAzure as dsaz
543
6
from cloudinit.util import find_freebsd_part
6
from cloudinit.util import find_freebsd_part
544
7
from cloudinit.util import get_path_dev_freebsd
7
from cloudinit.util import get_path_dev_freebsd
546
8
8
from cloudinit.version import version_string as vs
547
9
from cloudinit.tests.helpers import (CiTestCase, TestCase, populate_dir, mock,
9
from cloudinit.tests.helpers import (CiTestCase, TestCase, populate_dir, mock,
548
10
                                     ExitStack, PY26, SkipTest)
10
                                     ExitStack, PY26, SkipTest)
549
11
11
550
@@ -16,7 +16,8 @@ import xml.etree.ElementTree as ET
551
16
import yaml
16
import yaml
552
17
17
553
18
18
555
19
def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None):
19
def construct_valid_ovf_env(data=None, pubkeys=None,
556
20
                            userdata=None, platform_settings=None):
557
20
    if data is None:
21
    if data is None:
558
21
        data = {'HostName': 'FOOHOST'}
22
        data = {'HostName': 'FOOHOST'}
559
22
    if pubkeys is None:
23
    if pubkeys is None:
560
@@ -66,10 +67,12 @@ def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None):
561
66
  xmlns:i="http://www.w3.org/2001/XMLSchema-instance">
67
  xmlns:i="http://www.w3.org/2001/XMLSchema-instance">
562
67
 <KmsServerHostname>kms.core.windows.net</KmsServerHostname>
68
 <KmsServerHostname>kms.core.windows.net</KmsServerHostname>
563
68
 <ProvisionGuestAgent>false</ProvisionGuestAgent>
69
 <ProvisionGuestAgent>false</ProvisionGuestAgent>
568
69
 <GuestAgentPackageName i:nil="true" />
70
 <GuestAgentPackageName i:nil="true" />"""
569
70
 </PlatformSettings></wa:PlatformSettingsSection>
71
    if platform_settings:
570
71
</Environment>
72
        for k, v in platform_settings.items():
571
72
    """
73
            content += "<%s>%s</%s>\n" % (k, v, k)
572
74
    content += """</PlatformSettings></wa:PlatformSettingsSection>
573
75
</Environment>"""
574
73
76
575
74
    return content
77
    return content
576
75
78
577
@@ -1107,4 +1110,146 @@ class TestAzureNetExists(CiTestCase):
578
1107
        self.assertTrue(hasattr(dsaz, "DataSourceAzureNet"))
1110
        self.assertTrue(hasattr(dsaz, "DataSourceAzureNet"))
579
1108
1111
580
1109
1112
581
1113
@mock.patch('cloudinit.sources.DataSourceAzure.util.subp')
582
1114
@mock.patch.object(dsaz, 'get_hostname')
583
1115
@mock.patch.object(dsaz, 'set_hostname')
584
1116
class TestAzureDataSourcePreprovisioning(CiTestCase):
585
1117
586
1118
    def setUp(self):
587
1119
        super(TestAzureDataSourcePreprovisioning, self).setUp()
588
1120
        tmp = self.tmp_dir()
589
1121
        self.waagent_d = self.tmp_path('/var/lib/waagent', tmp)
590
1122
        self.paths = helpers.Paths({'cloud_dir': tmp})
591
1123
        dsaz.BUILTIN_DS_CONFIG['data_dir'] = self.waagent_d
592
1124
593
1125
    def test_read_azure_ovf_with_true_flag(self, *args):
594
1126
        """The read_azure_ovf method should set the PreprovisionedVM
595
1127
           cfg flag if the proper setting is present."""
596
1128
        content = construct_valid_ovf_env(
597
1129
            platform_settings={"PreprovisionedVm": "True"})
598
1130
        ret = dsaz.read_azure_ovf(content)
599
1131
        cfg = ret[2]
600
1132
        self.assertTrue(cfg['PreprovisionedVm'])
601
1133
602
1134
    def test_read_azure_ovf_with_false_flag(self, *args):
603
1135
        """The read_azure_ovf method should set the PreprovisionedVM
604
1136
           cfg flag to false if the proper setting is false."""
605
1137
        content = construct_valid_ovf_env(
606
1138
            platform_settings={"PreprovisionedVm": "False"})
607
1139
        ret = dsaz.read_azure_ovf(content)
608
1140
        cfg = ret[2]
609
1141
        self.assertFalse(cfg['PreprovisionedVm'])
610
1142
611
1143
    def test_read_azure_ovf_without_flag(self, *args):
612
1144
        """The read_azure_ovf method should not set the
613
1145
           PreprovisionedVM cfg flag."""
614
1146
        content = construct_valid_ovf_env()
615
1147
        ret = dsaz.read_azure_ovf(content)
616
1148
        cfg = ret[2]
617
1149
        self.assertFalse(cfg['PreprovisionedVm'])
618
1150
619
1151
    @mock.patch('cloudinit.sources.DataSourceAzure.util.is_FreeBSD')
620
1152
    @mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network')
621
1153
    @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
622
1154
    @mock.patch('requests.Session.request')
623
1155
    def test_poll_imds_returns_ovf_env(self, fake_resp, m_dhcp, m_net,
624
1156
                                       m_is_bsd, *args):
625
1157
        """The _poll_imds method should return the ovf_env.xml."""
626
1158
        m_is_bsd.return_value = False
627
1159
        m_dhcp.return_value = [{
628
1160
            'interface': 'eth9', 'fixed-address': '192.168.2.9',
629
1161
            'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0'}]
630
1162
        url = 'http://{0}/metadata/reprovisiondata?api-version=2017-04-02'
631
1163
        host = "169.254.169.254"
632
1164
        full_url = url.format(host)
633
1165
        fake_resp.return_value = mock.MagicMock(status_code=200, text="ovf")
634
1166
        dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths)
635
1167
        self.assertTrue(len(dsa._poll_imds()) > 0)
636
1168
        self.assertEqual(fake_resp.call_args_list,
637
1169
                         [mock.call(allow_redirects=True,
638
1170
                                    headers={'Metadata': 'true',
639
1171
                                             'User-Agent':
640
1172
                                             'Cloud-Init/%s' % vs()
641
1173
                                             }, method='GET', timeout=60.0,
642
1174
                                    url=full_url),
643
1175
                          mock.call(allow_redirects=True,
644
1176
                                    headers={'Metadata': 'true',
645
1177
                                             'User-Agent':
646
1178
                                             'Cloud-Init/%s' % vs()
647
1179
                                             }, method='GET', url=full_url)])
648
1180
        self.assertEqual(m_dhcp.call_count, 1)
649
1181
        m_net.assert_any_call(
650
1182
            broadcast='192.168.2.255', interface='eth9', ip='192.168.2.9',
651
1183
            prefix_or_mask='255.255.255.0', router='192.168.2.1')
652
1184
        self.assertEqual(m_net.call_count, 1)
653
1185
654
1186
    @mock.patch('cloudinit.sources.DataSourceAzure.util.is_FreeBSD')
655
1187
    @mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network')
656
1188
    @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
657
1189
    @mock.patch('requests.Session.request')
658
1190
    def test__reprovision_calls__poll_imds(self, fake_resp, m_dhcp, m_net,
659
1191
                                           m_is_bsd, *args):
660
1192
        """The _reprovision method should call poll IMDS."""
661
1193
        m_is_bsd.return_value = False
662
1194
        m_dhcp.return_value = [{
663
1195
            'interface': 'eth9', 'fixed-address': '192.168.2.9',
664
1196
            'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0',
665
1197
            'unknown-245': '624c3620'}]
666
1198
        url = 'http://{0}/metadata/reprovisiondata?api-version=2017-04-02'
667
1199
        host = "169.254.169.254"
668
1200
        full_url = url.format(host)
669
1201
        hostname = "myhost"
670
1202
        username = "myuser"
671
1203
        odata = {'HostName': hostname, 'UserName': username}
672
1204
        content = construct_valid_ovf_env(data=odata)
673
1205
        fake_resp.return_value = mock.MagicMock(status_code=200, text=content)
674
1206
        dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths)
675
1207
        md, ud, cfg, d = dsa._reprovision()
676
1208
        self.assertEqual(md['local-hostname'], hostname)
677
1209
        self.assertEqual(cfg['system_info']['default_user']['name'], username)
678
1210
        self.assertEqual(fake_resp.call_args_list,
679
1211
                         [mock.call(allow_redirects=True,
680
1212
                                    headers={'Metadata': 'true',
681
1213
                                             'User-Agent':
682
1214
                                             'Cloud-Init/%s' % vs()},
683
1215
                                    method='GET', timeout=60.0, url=full_url),
684
1216
                          mock.call(allow_redirects=True,
685
1217
                                    headers={'Metadata': 'true',
686
1218
                                             'User-Agent':
687
1219
                                             'Cloud-Init/%s' % vs()},
688
1220
                                    method='GET', url=full_url)])
689
1221
        self.assertEqual(m_dhcp.call_count, 1)
690
1222
        m_net.assert_any_call(
691
1223
            broadcast='192.168.2.255', interface='eth9', ip='192.168.2.9',
692
1224
            prefix_or_mask='255.255.255.0', router='192.168.2.1')
693
1225
        self.assertEqual(m_net.call_count, 1)
694
1226
695
1227
    @mock.patch('cloudinit.sources.DataSourceAzure.util.write_file')
696
1228
    @mock.patch('os.path.isfile')
697
1229
    def test__should_reprovision_with_true_cfg(self, isfile, write_f, *args):
698
1230
        """The _should_reprovision method should return true with config
699
1231
           flag present."""
700
1232
        isfile.return_value = False
701
1233
        dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths)
702
1234
        self.assertTrue(dsa._should_reprovision(
703
1235
            (None, None, {'PreprovisionedVm': True}, None)))
704
1236
705
1237
    @mock.patch('os.path.isfile')
706
1238
    def test__should_reprovision_with_file_existing(self, isfile, *args):
707
1239
        """The _should_reprovision method should return True if the sentinal
708
1240
           exists."""
709
1241
        isfile.return_value = True
710
1242
        dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths)
711
1243
        self.assertTrue(dsa._should_reprovision(
712
1244
            (None, None, {'preprovisionedvm': False}, None)))
713
1245
714
1246
    @mock.patch('os.path.isfile')
715
1247
    def test__should_reprovision_returns_false(self, isfile, *args):
716
1248
        """The _should_reprovision method should return False
717
1249
           if config and sentinal are not present."""
718
1250
        isfile.return_value = False
719
1251
        dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths)
720
1252
        self.assertFalse(dsa._should_reprovision((None, None, {}, None)))
721
1253
722
1254
723
1110
# vi: ts=4 expandtab
1255
# vi: ts=4 expandtab
724
diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
725
index f0dc833..0f7267b 100644
726
--- a/tests/unittests/test_datasource/test_ec2.py
727
+++ b/tests/unittests/test_datasource/test_ec2.py
728
@@ -425,7 +425,7 @@ class TestEc2(test_helpers.HttprettyTestCase):
729
425
            self.logs.getvalue())
425
            self.logs.getvalue())
730
426
426
731
427
    @httpretty.activate
427
    @httpretty.activate
733
428
    @mock.patch('cloudinit.net.EphemeralIPv4Network')
428
    @mock.patch('cloudinit.net.dhcp.EphemeralIPv4Network')
734
429
    @mock.patch('cloudinit.net.find_fallback_nic')
429
    @mock.patch('cloudinit.net.find_fallback_nic')
735
430
    @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
430
    @mock.patch('cloudinit.net.dhcp.maybe_perform_dhcp_discovery')
736
431
    @mock.patch('cloudinit.sources.DataSourceEc2.util.is_FreeBSD')
431
    @mock.patch('cloudinit.sources.DataSourceEc2.util.is_FreeBSD')
737
diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py
738
index ddea13d..ac33e8e 100644
739
--- a/tests/unittests/test_net.py
740
+++ b/tests/unittests/test_net.py
741
@@ -2948,4 +2948,16 @@ class TestRenameInterfaces(CiTestCase):
742
2948
        mock_subp.assert_has_calls(expected)
2948
        mock_subp.assert_has_calls(expected)
743
2949
2949
744
2950
2950
745
2951
class TestNetworkState(CiTestCase):
746
2952
747
2953
    def test_bcast_addr(self):
748
2954
        """Test mask_and_ipv4_to_bcast_addr proper execution."""
749
2955
        bcast_addr = network_state.mask_and_ipv4_to_bcast_addr
750
2956
        self.assertEqual("192.168.1.255",
751
2957
                         bcast_addr("255.255.255.0", "192.168.1.1"))
752
2958
        self.assertEqual("128.42.7.255",
753
2959
                         bcast_addr("255.255.248.0", "128.42.5.4"))
754
2960
        self.assertEqual("10.1.21.255",
755
2961
                         bcast_addr("255.255.255.0", "10.1.21.4"))
756
2962
757
2951
# vi: ts=4 expandtab
2963
# vi: ts=4 expandtab
Reviewer	Review Type	Date Requested	Status
Chad Smith			Approve on 2018-01-24
Server Team CI bot	continuous-integration		Approve on 2018-01-24
Scott Moser			Approve on 2018-01-22
Paul Meyer (community)		2017-11-28	Approve on 2017-12-01
Review via email: mp+334341@code.launchpad.net