Merge ~vtqanh/cloud-init:addKVPtelemetry into cloud-init:master
- Git
- lp:~vtqanh/cloud-init
- addKVPtelemetry
- Merge into master
Status: | Merged |
---|---|
Approved by: | Ryan Harper |
Approved revision: | d0778d381caad5b9edbdca013eb443652bb8c81b |
Merge reported by: | Server Team CI bot |
Merged at revision: | not available |
Proposed branch: | ~vtqanh/cloud-init:addKVPtelemetry |
Merge into: | cloud-init:master |
Diff against target: |
606 lines (+179/-83) 2 files modified
cloudinit/sources/DataSourceAzure.py (+148/-83) cloudinit/sources/helpers/azure.py (+31/-0) |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Server Team CI bot | continuous-integration | Approve | |
Ryan Harper | Approve | ||
Review via email: mp+365374@code.launchpad.net |
Commit message
DatasourceAzure: add additional logging for azure datasource
Create an Azure logging decorator and use additional ReportEventStack
context managers to provide additional logging details.
Description of the change
Jason Zions (jasonzio) wrote : | # |
Ryan Harper (raharper) wrote : | # |
I've pointed CI at this branch.
Server Team CI bot (server-team-bot) wrote : | # |
FAILED: Continuous integration, rev:d0778d381ca
https:/
Executed test runs:
FAILED: Checkout
Click here to trigger a rebuild:
https:/
Anh Vo (MSFT) (vtqanh) wrote : | # |
Looks like CI failed to checkout due to some authentication issue:
Is there any settings necessary on my branch to allow this
Cloning the remote Git repository
Cloning repository https:/
> git init /var/lib/
Fetching upstream changes from https:/
> git --version # timeout=10
> git fetch --tags --progress https:/
ERROR: Error cloning remote repo 'origin'
hudson.
stdout:
stderr: remote: Authorisation required.
Ryan Harper (raharper) wrote : | # |
Likely my fault, I'll submit again.
Server Team CI bot (server-team-bot) wrote : | # |
PASSED: Continuous integration, rev:d0778d381ca
https:/
Executed test runs:
SUCCESS: Checkout
SUCCESS: Unit & Style Tests
SUCCESS: Ubuntu LTS: Build
SUCCESS: Ubuntu LTS: Integration
IN_PROGRESS: Declarative: Post Actions
Click here to trigger a rebuild:
https:/
Ryan Harper (raharper) : | # |
Server Team CI bot (server-team-bot) : | # |
Preview Diff
1 | diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py | |||
2 | 0 | old mode 100644 | 0 | old mode 100644 |
3 | 1 | new mode 100755 | 1 | new mode 100755 |
4 | index b4e3f06..d4230b3 | |||
5 | --- a/cloudinit/sources/DataSourceAzure.py | |||
6 | +++ b/cloudinit/sources/DataSourceAzure.py | |||
7 | @@ -21,10 +21,14 @@ from cloudinit import net | |||
8 | 21 | from cloudinit.event import EventType | 21 | from cloudinit.event import EventType |
9 | 22 | from cloudinit.net.dhcp import EphemeralDHCPv4 | 22 | from cloudinit.net.dhcp import EphemeralDHCPv4 |
10 | 23 | from cloudinit import sources | 23 | from cloudinit import sources |
11 | 24 | from cloudinit.sources.helpers.azure import get_metadata_from_fabric | ||
12 | 25 | from cloudinit.sources.helpers import netlink | 24 | from cloudinit.sources.helpers import netlink |
13 | 26 | from cloudinit.url_helper import UrlError, readurl, retry_on_url_exc | 25 | from cloudinit.url_helper import UrlError, readurl, retry_on_url_exc |
14 | 27 | from cloudinit import util | 26 | from cloudinit import util |
15 | 27 | from cloudinit.reporting import events | ||
16 | 28 | |||
17 | 29 | from cloudinit.sources.helpers.azure import (azure_ds_reporter, | ||
18 | 30 | azure_ds_telemetry_reporter, | ||
19 | 31 | get_metadata_from_fabric) | ||
20 | 28 | 32 | ||
21 | 29 | LOG = logging.getLogger(__name__) | 33 | LOG = logging.getLogger(__name__) |
22 | 30 | 34 | ||
23 | @@ -244,6 +248,7 @@ def set_hostname(hostname, hostname_command='hostname'): | |||
24 | 244 | util.subp([hostname_command, hostname]) | 248 | util.subp([hostname_command, hostname]) |
25 | 245 | 249 | ||
26 | 246 | 250 | ||
27 | 251 | @azure_ds_telemetry_reporter | ||
28 | 247 | @contextlib.contextmanager | 252 | @contextlib.contextmanager |
29 | 248 | def temporary_hostname(temp_hostname, cfg, hostname_command='hostname'): | 253 | def temporary_hostname(temp_hostname, cfg, hostname_command='hostname'): |
30 | 249 | """ | 254 | """ |
31 | @@ -290,6 +295,7 @@ class DataSourceAzure(sources.DataSource): | |||
32 | 290 | root = sources.DataSource.__str__(self) | 295 | root = sources.DataSource.__str__(self) |
33 | 291 | return "%s [seed=%s]" % (root, self.seed) | 296 | return "%s [seed=%s]" % (root, self.seed) |
34 | 292 | 297 | ||
35 | 298 | @azure_ds_telemetry_reporter | ||
36 | 293 | def bounce_network_with_azure_hostname(self): | 299 | def bounce_network_with_azure_hostname(self): |
37 | 294 | # When using cloud-init to provision, we have to set the hostname from | 300 | # When using cloud-init to provision, we have to set the hostname from |
38 | 295 | # the metadata and "bounce" the network to force DDNS to update via | 301 | # the metadata and "bounce" the network to force DDNS to update via |
39 | @@ -315,6 +321,7 @@ class DataSourceAzure(sources.DataSource): | |||
40 | 315 | util.logexc(LOG, "handling set_hostname failed") | 321 | util.logexc(LOG, "handling set_hostname failed") |
41 | 316 | return False | 322 | return False |
42 | 317 | 323 | ||
43 | 324 | @azure_ds_telemetry_reporter | ||
44 | 318 | def get_metadata_from_agent(self): | 325 | def get_metadata_from_agent(self): |
45 | 319 | temp_hostname = self.metadata.get('local-hostname') | 326 | temp_hostname = self.metadata.get('local-hostname') |
46 | 320 | agent_cmd = self.ds_cfg['agent_command'] | 327 | agent_cmd = self.ds_cfg['agent_command'] |
47 | @@ -344,15 +351,18 @@ class DataSourceAzure(sources.DataSource): | |||
48 | 344 | LOG.debug("ssh authentication: " | 351 | LOG.debug("ssh authentication: " |
49 | 345 | "using fingerprint from fabirc") | 352 | "using fingerprint from fabirc") |
50 | 346 | 353 | ||
60 | 347 | # wait very long for public SSH keys to arrive | 354 | with events.ReportEventStack( |
61 | 348 | # https://bugs.launchpad.net/cloud-init/+bug/1717611 | 355 | name="waiting-for-ssh-public-key", |
62 | 349 | missing = util.log_time(logfunc=LOG.debug, | 356 | description="wait for agents to retrieve ssh keys", |
63 | 350 | msg="waiting for SSH public key files", | 357 | parent=azure_ds_reporter): |
64 | 351 | func=util.wait_for_files, | 358 | # wait very long for public SSH keys to arrive |
65 | 352 | args=(fp_files, 900)) | 359 | # https://bugs.launchpad.net/cloud-init/+bug/1717611 |
66 | 353 | 360 | missing = util.log_time(logfunc=LOG.debug, | |
67 | 354 | if len(missing): | 361 | msg="waiting for SSH public key files", |
68 | 355 | LOG.warning("Did not find files, but going on: %s", missing) | 362 | func=util.wait_for_files, |
69 | 363 | args=(fp_files, 900)) | ||
70 | 364 | if len(missing): | ||
71 | 365 | LOG.warning("Did not find files, but going on: %s", missing) | ||
72 | 356 | 366 | ||
73 | 357 | metadata = {} | 367 | metadata = {} |
74 | 358 | metadata['public-keys'] = key_value or pubkeys_from_crt_files(fp_files) | 368 | metadata['public-keys'] = key_value or pubkeys_from_crt_files(fp_files) |
75 | @@ -366,6 +376,7 @@ class DataSourceAzure(sources.DataSource): | |||
76 | 366 | subplatform_type = 'seed-dir' | 376 | subplatform_type = 'seed-dir' |
77 | 367 | return '%s (%s)' % (subplatform_type, self.seed) | 377 | return '%s (%s)' % (subplatform_type, self.seed) |
78 | 368 | 378 | ||
79 | 379 | @azure_ds_telemetry_reporter | ||
80 | 369 | def crawl_metadata(self): | 380 | def crawl_metadata(self): |
81 | 370 | """Walk all instance metadata sources returning a dict on success. | 381 | """Walk all instance metadata sources returning a dict on success. |
82 | 371 | 382 | ||
83 | @@ -467,6 +478,7 @@ class DataSourceAzure(sources.DataSource): | |||
84 | 467 | super(DataSourceAzure, self).clear_cached_attrs(attr_defaults) | 478 | super(DataSourceAzure, self).clear_cached_attrs(attr_defaults) |
85 | 468 | self._metadata_imds = sources.UNSET | 479 | self._metadata_imds = sources.UNSET |
86 | 469 | 480 | ||
87 | 481 | @azure_ds_telemetry_reporter | ||
88 | 470 | def _get_data(self): | 482 | def _get_data(self): |
89 | 471 | """Crawl and process datasource metadata caching metadata as attrs. | 483 | """Crawl and process datasource metadata caching metadata as attrs. |
90 | 472 | 484 | ||
91 | @@ -513,6 +525,7 @@ class DataSourceAzure(sources.DataSource): | |||
92 | 513 | # quickly (local check only) if self.instance_id is still valid | 525 | # quickly (local check only) if self.instance_id is still valid |
93 | 514 | return sources.instance_id_matches_system_uuid(self.get_instance_id()) | 526 | return sources.instance_id_matches_system_uuid(self.get_instance_id()) |
94 | 515 | 527 | ||
95 | 528 | @azure_ds_telemetry_reporter | ||
96 | 516 | def setup(self, is_new_instance): | 529 | def setup(self, is_new_instance): |
97 | 517 | if self._negotiated is False: | 530 | if self._negotiated is False: |
98 | 518 | LOG.debug("negotiating for %s (new_instance=%s)", | 531 | LOG.debug("negotiating for %s (new_instance=%s)", |
99 | @@ -580,6 +593,7 @@ class DataSourceAzure(sources.DataSource): | |||
100 | 580 | if nl_sock: | 593 | if nl_sock: |
101 | 581 | nl_sock.close() | 594 | nl_sock.close() |
102 | 582 | 595 | ||
103 | 596 | @azure_ds_telemetry_reporter | ||
104 | 583 | def _report_ready(self, lease): | 597 | def _report_ready(self, lease): |
105 | 584 | """Tells the fabric provisioning has completed """ | 598 | """Tells the fabric provisioning has completed """ |
106 | 585 | try: | 599 | try: |
107 | @@ -617,9 +631,14 @@ class DataSourceAzure(sources.DataSource): | |||
108 | 617 | def _reprovision(self): | 631 | def _reprovision(self): |
109 | 618 | """Initiate the reprovisioning workflow.""" | 632 | """Initiate the reprovisioning workflow.""" |
110 | 619 | contents = self._poll_imds() | 633 | contents = self._poll_imds() |
114 | 620 | md, ud, cfg = read_azure_ovf(contents) | 634 | with events.ReportEventStack( |
115 | 621 | return (md, ud, cfg, {'ovf-env.xml': contents}) | 635 | name="reprovisioning-read-azure-ovf", |
116 | 622 | 636 | description="read azure ovf during reprovisioning", | |
117 | 637 | parent=azure_ds_reporter): | ||
118 | 638 | md, ud, cfg = read_azure_ovf(contents) | ||
119 | 639 | return (md, ud, cfg, {'ovf-env.xml': contents}) | ||
120 | 640 | |||
121 | 641 | @azure_ds_telemetry_reporter | ||
122 | 623 | def _negotiate(self): | 642 | def _negotiate(self): |
123 | 624 | """Negotiate with fabric and return data from it. | 643 | """Negotiate with fabric and return data from it. |
124 | 625 | 644 | ||
125 | @@ -652,6 +671,7 @@ class DataSourceAzure(sources.DataSource): | |||
126 | 652 | util.del_file(REPROVISION_MARKER_FILE) | 671 | util.del_file(REPROVISION_MARKER_FILE) |
127 | 653 | return fabric_data | 672 | return fabric_data |
128 | 654 | 673 | ||
129 | 674 | @azure_ds_telemetry_reporter | ||
130 | 655 | def activate(self, cfg, is_new_instance): | 675 | def activate(self, cfg, is_new_instance): |
131 | 656 | address_ephemeral_resize(is_new_instance=is_new_instance, | 676 | address_ephemeral_resize(is_new_instance=is_new_instance, |
132 | 657 | preserve_ntfs=self.ds_cfg.get( | 677 | preserve_ntfs=self.ds_cfg.get( |
133 | @@ -690,12 +710,14 @@ def _partitions_on_device(devpath, maxnum=16): | |||
134 | 690 | return [] | 710 | return [] |
135 | 691 | 711 | ||
136 | 692 | 712 | ||
137 | 713 | @azure_ds_telemetry_reporter | ||
138 | 693 | def _has_ntfs_filesystem(devpath): | 714 | def _has_ntfs_filesystem(devpath): |
139 | 694 | ntfs_devices = util.find_devs_with("TYPE=ntfs", no_cache=True) | 715 | ntfs_devices = util.find_devs_with("TYPE=ntfs", no_cache=True) |
140 | 695 | LOG.debug('ntfs_devices found = %s', ntfs_devices) | 716 | LOG.debug('ntfs_devices found = %s', ntfs_devices) |
141 | 696 | return os.path.realpath(devpath) in ntfs_devices | 717 | return os.path.realpath(devpath) in ntfs_devices |
142 | 697 | 718 | ||
143 | 698 | 719 | ||
144 | 720 | @azure_ds_telemetry_reporter | ||
145 | 699 | def can_dev_be_reformatted(devpath, preserve_ntfs): | 721 | def can_dev_be_reformatted(devpath, preserve_ntfs): |
146 | 700 | """Determine if the ephemeral drive at devpath should be reformatted. | 722 | """Determine if the ephemeral drive at devpath should be reformatted. |
147 | 701 | 723 | ||
148 | @@ -744,43 +766,59 @@ def can_dev_be_reformatted(devpath, preserve_ntfs): | |||
149 | 744 | (cand_part, cand_path, devpath)) | 766 | (cand_part, cand_path, devpath)) |
150 | 745 | return False, msg | 767 | return False, msg |
151 | 746 | 768 | ||
152 | 769 | @azure_ds_telemetry_reporter | ||
153 | 747 | def count_files(mp): | 770 | def count_files(mp): |
154 | 748 | ignored = set(['dataloss_warning_readme.txt']) | 771 | ignored = set(['dataloss_warning_readme.txt']) |
155 | 749 | return len([f for f in os.listdir(mp) if f.lower() not in ignored]) | 772 | return len([f for f in os.listdir(mp) if f.lower() not in ignored]) |
156 | 750 | 773 | ||
157 | 751 | bmsg = ('partition %s (%s) on device %s was ntfs formatted' % | 774 | bmsg = ('partition %s (%s) on device %s was ntfs formatted' % |
158 | 752 | (cand_part, cand_path, devpath)) | 775 | (cand_part, cand_path, devpath)) |
175 | 753 | try: | 776 | |
176 | 754 | file_count = util.mount_cb(cand_path, count_files, mtype="ntfs", | 777 | with events.ReportEventStack( |
177 | 755 | update_env_for_mount={'LANG': 'C'}) | 778 | name="mount-ntfs-and-count", |
178 | 756 | except util.MountFailedError as e: | 779 | description="mount-ntfs-and-count", |
179 | 757 | if "unknown filesystem type 'ntfs'" in str(e): | 780 | parent=azure_ds_reporter) as evt: |
180 | 758 | return True, (bmsg + ' but this system cannot mount NTFS,' | 781 | try: |
181 | 759 | ' assuming there are no important files.' | 782 | file_count = util.mount_cb(cand_path, count_files, mtype="ntfs", |
182 | 760 | ' Formatting allowed.') | 783 | update_env_for_mount={'LANG': 'C'}) |
183 | 761 | return False, bmsg + ' but mount of %s failed: %s' % (cand_part, e) | 784 | except util.MountFailedError as e: |
184 | 762 | 785 | evt.description = "cannot mount ntfs" | |
185 | 763 | if file_count != 0: | 786 | if "unknown filesystem type 'ntfs'" in str(e): |
186 | 764 | LOG.warning("it looks like you're using NTFS on the ephemeral disk, " | 787 | return True, (bmsg + ' but this system cannot mount NTFS,' |
187 | 765 | 'to ensure that filesystem does not get wiped, set ' | 788 | ' assuming there are no important files.' |
188 | 766 | '%s.%s in config', '.'.join(DS_CFG_PATH), | 789 | ' Formatting allowed.') |
189 | 767 | DS_CFG_KEY_PRESERVE_NTFS) | 790 | return False, bmsg + ' but mount of %s failed: %s' % (cand_part, e) |
190 | 768 | return False, bmsg + ' but had %d files on it.' % file_count | 791 | |
191 | 792 | if file_count != 0: | ||
192 | 793 | evt.description = "mounted and counted %d files" % file_count | ||
193 | 794 | LOG.warning("it looks like you're using NTFS on the ephemeral" | ||
194 | 795 | " disk, to ensure that filesystem does not get wiped," | ||
195 | 796 | " set %s.%s in config", '.'.join(DS_CFG_PATH), | ||
196 | 797 | DS_CFG_KEY_PRESERVE_NTFS) | ||
197 | 798 | return False, bmsg + ' but had %d files on it.' % file_count | ||
198 | 769 | 799 | ||
199 | 770 | return True, bmsg + ' and had no important files. Safe for reformatting.' | 800 | return True, bmsg + ' and had no important files. Safe for reformatting.' |
200 | 771 | 801 | ||
201 | 772 | 802 | ||
202 | 803 | @azure_ds_telemetry_reporter | ||
203 | 773 | def address_ephemeral_resize(devpath=RESOURCE_DISK_PATH, maxwait=120, | 804 | def address_ephemeral_resize(devpath=RESOURCE_DISK_PATH, maxwait=120, |
204 | 774 | is_new_instance=False, preserve_ntfs=False): | 805 | is_new_instance=False, preserve_ntfs=False): |
205 | 775 | # wait for ephemeral disk to come up | 806 | # wait for ephemeral disk to come up |
206 | 776 | naplen = .2 | 807 | naplen = .2 |
214 | 777 | missing = util.wait_for_files([devpath], maxwait=maxwait, naplen=naplen, | 808 | with events.ReportEventStack( |
215 | 778 | log_pre="Azure ephemeral disk: ") | 809 | name="wait-for-ephemeral-disk", |
216 | 779 | 810 | description="wait for ephemeral disk", | |
217 | 780 | if missing: | 811 | parent=azure_ds_reporter): |
218 | 781 | LOG.warning("ephemeral device '%s' did not appear after %d seconds.", | 812 | missing = util.wait_for_files([devpath], |
219 | 782 | devpath, maxwait) | 813 | maxwait=maxwait, |
220 | 783 | return | 814 | naplen=naplen, |
221 | 815 | log_pre="Azure ephemeral disk: ") | ||
222 | 816 | |||
223 | 817 | if missing: | ||
224 | 818 | LOG.warning("ephemeral device '%s' did" | ||
225 | 819 | " not appear after %d seconds.", | ||
226 | 820 | devpath, maxwait) | ||
227 | 821 | return | ||
228 | 784 | 822 | ||
229 | 785 | result = False | 823 | result = False |
230 | 786 | msg = None | 824 | msg = None |
231 | @@ -808,6 +846,7 @@ def address_ephemeral_resize(devpath=RESOURCE_DISK_PATH, maxwait=120, | |||
232 | 808 | return | 846 | return |
233 | 809 | 847 | ||
234 | 810 | 848 | ||
235 | 849 | @azure_ds_telemetry_reporter | ||
236 | 811 | def perform_hostname_bounce(hostname, cfg, prev_hostname): | 850 | def perform_hostname_bounce(hostname, cfg, prev_hostname): |
237 | 812 | # set the hostname to 'hostname' if it is not already set to that. | 851 | # set the hostname to 'hostname' if it is not already set to that. |
238 | 813 | # then, if policy is not off, bounce the interface using command | 852 | # then, if policy is not off, bounce the interface using command |
239 | @@ -843,6 +882,7 @@ def perform_hostname_bounce(hostname, cfg, prev_hostname): | |||
240 | 843 | return True | 882 | return True |
241 | 844 | 883 | ||
242 | 845 | 884 | ||
243 | 885 | @azure_ds_telemetry_reporter | ||
244 | 846 | def crtfile_to_pubkey(fname, data=None): | 886 | def crtfile_to_pubkey(fname, data=None): |
245 | 847 | pipeline = ('openssl x509 -noout -pubkey < "$0" |' | 887 | pipeline = ('openssl x509 -noout -pubkey < "$0" |' |
246 | 848 | 'ssh-keygen -i -m PKCS8 -f /dev/stdin') | 888 | 'ssh-keygen -i -m PKCS8 -f /dev/stdin') |
247 | @@ -851,6 +891,7 @@ def crtfile_to_pubkey(fname, data=None): | |||
248 | 851 | return out.rstrip() | 891 | return out.rstrip() |
249 | 852 | 892 | ||
250 | 853 | 893 | ||
251 | 894 | @azure_ds_telemetry_reporter | ||
252 | 854 | def pubkeys_from_crt_files(flist): | 895 | def pubkeys_from_crt_files(flist): |
253 | 855 | pubkeys = [] | 896 | pubkeys = [] |
254 | 856 | errors = [] | 897 | errors = [] |
255 | @@ -866,6 +907,7 @@ def pubkeys_from_crt_files(flist): | |||
256 | 866 | return pubkeys | 907 | return pubkeys |
257 | 867 | 908 | ||
258 | 868 | 909 | ||
259 | 910 | @azure_ds_telemetry_reporter | ||
260 | 869 | def write_files(datadir, files, dirmode=None): | 911 | def write_files(datadir, files, dirmode=None): |
261 | 870 | 912 | ||
262 | 871 | def _redact_password(cnt, fname): | 913 | def _redact_password(cnt, fname): |
263 | @@ -893,6 +935,7 @@ def write_files(datadir, files, dirmode=None): | |||
264 | 893 | util.write_file(filename=fname, content=content, mode=0o600) | 935 | util.write_file(filename=fname, content=content, mode=0o600) |
265 | 894 | 936 | ||
266 | 895 | 937 | ||
267 | 938 | @azure_ds_telemetry_reporter | ||
268 | 896 | def invoke_agent(cmd): | 939 | def invoke_agent(cmd): |
269 | 897 | # this is a function itself to simplify patching it for test | 940 | # this is a function itself to simplify patching it for test |
270 | 898 | if cmd: | 941 | if cmd: |
271 | @@ -912,6 +955,7 @@ def find_child(node, filter_func): | |||
272 | 912 | return ret | 955 | return ret |
273 | 913 | 956 | ||
274 | 914 | 957 | ||
275 | 958 | @azure_ds_telemetry_reporter | ||
276 | 915 | def load_azure_ovf_pubkeys(sshnode): | 959 | def load_azure_ovf_pubkeys(sshnode): |
277 | 916 | # This parses a 'SSH' node formatted like below, and returns | 960 | # This parses a 'SSH' node formatted like below, and returns |
278 | 917 | # an array of dicts. | 961 | # an array of dicts. |
279 | @@ -964,6 +1008,7 @@ def load_azure_ovf_pubkeys(sshnode): | |||
280 | 964 | return found | 1008 | return found |
281 | 965 | 1009 | ||
282 | 966 | 1010 | ||
283 | 1011 | @azure_ds_telemetry_reporter | ||
284 | 967 | def read_azure_ovf(contents): | 1012 | def read_azure_ovf(contents): |
285 | 968 | try: | 1013 | try: |
286 | 969 | dom = minidom.parseString(contents) | 1014 | dom = minidom.parseString(contents) |
287 | @@ -1064,6 +1109,7 @@ def read_azure_ovf(contents): | |||
288 | 1064 | return (md, ud, cfg) | 1109 | return (md, ud, cfg) |
289 | 1065 | 1110 | ||
290 | 1066 | 1111 | ||
291 | 1112 | @azure_ds_telemetry_reporter | ||
292 | 1067 | def _extract_preprovisioned_vm_setting(dom): | 1113 | def _extract_preprovisioned_vm_setting(dom): |
293 | 1068 | """Read the preprovision flag from the ovf. It should not | 1114 | """Read the preprovision flag from the ovf. It should not |
294 | 1069 | exist unless true.""" | 1115 | exist unless true.""" |
295 | @@ -1092,6 +1138,7 @@ def encrypt_pass(password, salt_id="$6$"): | |||
296 | 1092 | return crypt.crypt(password, salt_id + util.rand_str(strlen=16)) | 1138 | return crypt.crypt(password, salt_id + util.rand_str(strlen=16)) |
297 | 1093 | 1139 | ||
298 | 1094 | 1140 | ||
299 | 1141 | @azure_ds_telemetry_reporter | ||
300 | 1095 | def _check_freebsd_cdrom(cdrom_dev): | 1142 | def _check_freebsd_cdrom(cdrom_dev): |
301 | 1096 | """Return boolean indicating path to cdrom device has content.""" | 1143 | """Return boolean indicating path to cdrom device has content.""" |
302 | 1097 | try: | 1144 | try: |
303 | @@ -1103,6 +1150,7 @@ def _check_freebsd_cdrom(cdrom_dev): | |||
304 | 1103 | return False | 1150 | return False |
305 | 1104 | 1151 | ||
306 | 1105 | 1152 | ||
307 | 1153 | @azure_ds_telemetry_reporter | ||
308 | 1106 | def _get_random_seed(source=PLATFORM_ENTROPY_SOURCE): | 1154 | def _get_random_seed(source=PLATFORM_ENTROPY_SOURCE): |
309 | 1107 | """Return content random seed file if available, otherwise, | 1155 | """Return content random seed file if available, otherwise, |
310 | 1108 | return None.""" | 1156 | return None.""" |
311 | @@ -1126,6 +1174,7 @@ def _get_random_seed(source=PLATFORM_ENTROPY_SOURCE): | |||
312 | 1126 | return seed | 1174 | return seed |
313 | 1127 | 1175 | ||
314 | 1128 | 1176 | ||
315 | 1177 | @azure_ds_telemetry_reporter | ||
316 | 1129 | def list_possible_azure_ds_devs(): | 1178 | def list_possible_azure_ds_devs(): |
317 | 1130 | devlist = [] | 1179 | devlist = [] |
318 | 1131 | if util.is_FreeBSD(): | 1180 | if util.is_FreeBSD(): |
319 | @@ -1140,6 +1189,7 @@ def list_possible_azure_ds_devs(): | |||
320 | 1140 | return devlist | 1189 | return devlist |
321 | 1141 | 1190 | ||
322 | 1142 | 1191 | ||
323 | 1192 | @azure_ds_telemetry_reporter | ||
324 | 1143 | def load_azure_ds_dir(source_dir): | 1193 | def load_azure_ds_dir(source_dir): |
325 | 1144 | ovf_file = os.path.join(source_dir, "ovf-env.xml") | 1194 | ovf_file = os.path.join(source_dir, "ovf-env.xml") |
326 | 1145 | 1195 | ||
327 | @@ -1162,47 +1212,54 @@ def parse_network_config(imds_metadata): | |||
328 | 1162 | @param: imds_metadata: Dict of content read from IMDS network service. | 1212 | @param: imds_metadata: Dict of content read from IMDS network service. |
329 | 1163 | @return: Dictionary containing network version 2 standard configuration. | 1213 | @return: Dictionary containing network version 2 standard configuration. |
330 | 1164 | """ | 1214 | """ |
370 | 1165 | if imds_metadata != sources.UNSET and imds_metadata: | 1215 | with events.ReportEventStack( |
371 | 1166 | netconfig = {'version': 2, 'ethernets': {}} | 1216 | name="parse_network_config", |
372 | 1167 | LOG.debug('Azure: generating network configuration from IMDS') | 1217 | description="", |
373 | 1168 | network_metadata = imds_metadata['network'] | 1218 | parent=azure_ds_reporter) as evt: |
374 | 1169 | for idx, intf in enumerate(network_metadata['interface']): | 1219 | if imds_metadata != sources.UNSET and imds_metadata: |
375 | 1170 | nicname = 'eth{idx}'.format(idx=idx) | 1220 | netconfig = {'version': 2, 'ethernets': {}} |
376 | 1171 | dev_config = {} | 1221 | LOG.debug('Azure: generating network configuration from IMDS') |
377 | 1172 | for addr4 in intf['ipv4']['ipAddress']: | 1222 | network_metadata = imds_metadata['network'] |
378 | 1173 | privateIpv4 = addr4['privateIpAddress'] | 1223 | for idx, intf in enumerate(network_metadata['interface']): |
379 | 1174 | if privateIpv4: | 1224 | nicname = 'eth{idx}'.format(idx=idx) |
380 | 1175 | if dev_config.get('dhcp4', False): | 1225 | dev_config = {} |
381 | 1176 | # Append static address config for nic > 1 | 1226 | for addr4 in intf['ipv4']['ipAddress']: |
382 | 1177 | netPrefix = intf['ipv4']['subnet'][0].get( | 1227 | privateIpv4 = addr4['privateIpAddress'] |
383 | 1178 | 'prefix', '24') | 1228 | if privateIpv4: |
384 | 1179 | if not dev_config.get('addresses'): | 1229 | if dev_config.get('dhcp4', False): |
385 | 1180 | dev_config['addresses'] = [] | 1230 | # Append static address config for nic > 1 |
386 | 1181 | dev_config['addresses'].append( | 1231 | netPrefix = intf['ipv4']['subnet'][0].get( |
387 | 1182 | '{ip}/{prefix}'.format( | 1232 | 'prefix', '24') |
388 | 1183 | ip=privateIpv4, prefix=netPrefix)) | 1233 | if not dev_config.get('addresses'): |
389 | 1184 | else: | 1234 | dev_config['addresses'] = [] |
390 | 1185 | dev_config['dhcp4'] = True | 1235 | dev_config['addresses'].append( |
391 | 1186 | for addr6 in intf['ipv6']['ipAddress']: | 1236 | '{ip}/{prefix}'.format( |
392 | 1187 | privateIpv6 = addr6['privateIpAddress'] | 1237 | ip=privateIpv4, prefix=netPrefix)) |
393 | 1188 | if privateIpv6: | 1238 | else: |
394 | 1189 | dev_config['dhcp6'] = True | 1239 | dev_config['dhcp4'] = True |
395 | 1190 | break | 1240 | for addr6 in intf['ipv6']['ipAddress']: |
396 | 1191 | if dev_config: | 1241 | privateIpv6 = addr6['privateIpAddress'] |
397 | 1192 | mac = ':'.join(re.findall(r'..', intf['macAddress'])) | 1242 | if privateIpv6: |
398 | 1193 | dev_config.update( | 1243 | dev_config['dhcp6'] = True |
399 | 1194 | {'match': {'macaddress': mac.lower()}, | 1244 | break |
400 | 1195 | 'set-name': nicname}) | 1245 | if dev_config: |
401 | 1196 | netconfig['ethernets'][nicname] = dev_config | 1246 | mac = ':'.join(re.findall(r'..', intf['macAddress'])) |
402 | 1197 | else: | 1247 | dev_config.update( |
403 | 1198 | blacklist = ['mlx4_core'] | 1248 | {'match': {'macaddress': mac.lower()}, |
404 | 1199 | LOG.debug('Azure: generating fallback configuration') | 1249 | 'set-name': nicname}) |
405 | 1200 | # generate a network config, blacklist picking mlx4_core devs | 1250 | netconfig['ethernets'][nicname] = dev_config |
406 | 1201 | netconfig = net.generate_fallback_config( | 1251 | evt.description = "network config from imds" |
407 | 1202 | blacklist_drivers=blacklist, config_driver=True) | 1252 | else: |
408 | 1203 | return netconfig | 1253 | blacklist = ['mlx4_core'] |
409 | 1254 | LOG.debug('Azure: generating fallback configuration') | ||
410 | 1255 | # generate a network config, blacklist picking mlx4_core devs | ||
411 | 1256 | netconfig = net.generate_fallback_config( | ||
412 | 1257 | blacklist_drivers=blacklist, config_driver=True) | ||
413 | 1258 | evt.description = "network config from fallback" | ||
414 | 1259 | return netconfig | ||
415 | 1204 | 1260 | ||
416 | 1205 | 1261 | ||
417 | 1262 | @azure_ds_telemetry_reporter | ||
418 | 1206 | def get_metadata_from_imds(fallback_nic, retries): | 1263 | def get_metadata_from_imds(fallback_nic, retries): |
419 | 1207 | """Query Azure's network metadata service, returning a dictionary. | 1264 | """Query Azure's network metadata service, returning a dictionary. |
420 | 1208 | 1265 | ||
421 | @@ -1227,6 +1284,7 @@ def get_metadata_from_imds(fallback_nic, retries): | |||
422 | 1227 | return util.log_time(**kwargs) | 1284 | return util.log_time(**kwargs) |
423 | 1228 | 1285 | ||
424 | 1229 | 1286 | ||
425 | 1287 | @azure_ds_telemetry_reporter | ||
426 | 1230 | def _get_metadata_from_imds(retries): | 1288 | def _get_metadata_from_imds(retries): |
427 | 1231 | 1289 | ||
428 | 1232 | url = IMDS_URL + "instance?api-version=2017-12-01" | 1290 | url = IMDS_URL + "instance?api-version=2017-12-01" |
429 | @@ -1246,6 +1304,7 @@ def _get_metadata_from_imds(retries): | |||
430 | 1246 | return {} | 1304 | return {} |
431 | 1247 | 1305 | ||
432 | 1248 | 1306 | ||
433 | 1307 | @azure_ds_telemetry_reporter | ||
434 | 1249 | def maybe_remove_ubuntu_network_config_scripts(paths=None): | 1308 | def maybe_remove_ubuntu_network_config_scripts(paths=None): |
435 | 1250 | """Remove Azure-specific ubuntu network config for non-primary nics. | 1309 | """Remove Azure-specific ubuntu network config for non-primary nics. |
436 | 1251 | 1310 | ||
437 | @@ -1283,14 +1342,20 @@ def maybe_remove_ubuntu_network_config_scripts(paths=None): | |||
438 | 1283 | 1342 | ||
439 | 1284 | 1343 | ||
440 | 1285 | def _is_platform_viable(seed_dir): | 1344 | def _is_platform_viable(seed_dir): |
449 | 1286 | """Check platform environment to report if this datasource may run.""" | 1345 | with events.ReportEventStack( |
450 | 1287 | asset_tag = util.read_dmi_data('chassis-asset-tag') | 1346 | name="check-platform-viability", |
451 | 1288 | if asset_tag == AZURE_CHASSIS_ASSET_TAG: | 1347 | description="found azure asset tag", |
452 | 1289 | return True | 1348 | parent=azure_ds_reporter) as evt: |
453 | 1290 | LOG.debug("Non-Azure DMI asset tag '%s' discovered.", asset_tag) | 1349 | |
454 | 1291 | if os.path.exists(os.path.join(seed_dir, 'ovf-env.xml')): | 1350 | """Check platform environment to report if this datasource may run.""" |
455 | 1292 | return True | 1351 | asset_tag = util.read_dmi_data('chassis-asset-tag') |
456 | 1293 | return False | 1352 | if asset_tag == AZURE_CHASSIS_ASSET_TAG: |
457 | 1353 | return True | ||
458 | 1354 | LOG.debug("Non-Azure DMI asset tag '%s' discovered.", asset_tag) | ||
459 | 1355 | evt.description = "Non-Azure DMI asset tag '%s' discovered.", asset_tag | ||
460 | 1356 | if os.path.exists(os.path.join(seed_dir, 'ovf-env.xml')): | ||
461 | 1357 | return True | ||
462 | 1358 | return False | ||
463 | 1294 | 1359 | ||
464 | 1295 | 1360 | ||
465 | 1296 | class BrokenAzureDataSource(Exception): | 1361 | class BrokenAzureDataSource(Exception): |
466 | diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py | |||
467 | 1297 | old mode 100644 | 1362 | old mode 100644 |
468 | 1298 | new mode 100755 | 1363 | new mode 100755 |
469 | index 2829dd2..d3af05e | |||
470 | --- a/cloudinit/sources/helpers/azure.py | |||
471 | +++ b/cloudinit/sources/helpers/azure.py | |||
472 | @@ -16,10 +16,27 @@ from xml.etree import ElementTree | |||
473 | 16 | 16 | ||
474 | 17 | from cloudinit import url_helper | 17 | from cloudinit import url_helper |
475 | 18 | from cloudinit import util | 18 | from cloudinit import util |
476 | 19 | from cloudinit.reporting import events | ||
477 | 19 | 20 | ||
478 | 20 | LOG = logging.getLogger(__name__) | 21 | LOG = logging.getLogger(__name__) |
479 | 21 | 22 | ||
480 | 22 | 23 | ||
481 | 24 | azure_ds_reporter = events.ReportEventStack( | ||
482 | 25 | name="azure-ds", | ||
483 | 26 | description="initialize reporter for azure ds", | ||
484 | 27 | reporting_enabled=True) | ||
485 | 28 | |||
486 | 29 | |||
487 | 30 | def azure_ds_telemetry_reporter(func): | ||
488 | 31 | def impl(*args, **kwargs): | ||
489 | 32 | with events.ReportEventStack( | ||
490 | 33 | name=func.__name__, | ||
491 | 34 | description=func.__name__, | ||
492 | 35 | parent=azure_ds_reporter): | ||
493 | 36 | return func(*args, **kwargs) | ||
494 | 37 | return impl | ||
495 | 38 | |||
496 | 39 | |||
497 | 23 | @contextmanager | 40 | @contextmanager |
498 | 24 | def cd(newdir): | 41 | def cd(newdir): |
499 | 25 | prevdir = os.getcwd() | 42 | prevdir = os.getcwd() |
500 | @@ -119,6 +136,7 @@ class OpenSSLManager(object): | |||
501 | 119 | def clean_up(self): | 136 | def clean_up(self): |
502 | 120 | util.del_dir(self.tmpdir) | 137 | util.del_dir(self.tmpdir) |
503 | 121 | 138 | ||
504 | 139 | @azure_ds_telemetry_reporter | ||
505 | 122 | def generate_certificate(self): | 140 | def generate_certificate(self): |
506 | 123 | LOG.debug('Generating certificate for communication with fabric...') | 141 | LOG.debug('Generating certificate for communication with fabric...') |
507 | 124 | if self.certificate is not None: | 142 | if self.certificate is not None: |
508 | @@ -139,17 +157,20 @@ class OpenSSLManager(object): | |||
509 | 139 | LOG.debug('New certificate generated.') | 157 | LOG.debug('New certificate generated.') |
510 | 140 | 158 | ||
511 | 141 | @staticmethod | 159 | @staticmethod |
512 | 160 | @azure_ds_telemetry_reporter | ||
513 | 142 | def _run_x509_action(action, cert): | 161 | def _run_x509_action(action, cert): |
514 | 143 | cmd = ['openssl', 'x509', '-noout', action] | 162 | cmd = ['openssl', 'x509', '-noout', action] |
515 | 144 | result, _ = util.subp(cmd, data=cert) | 163 | result, _ = util.subp(cmd, data=cert) |
516 | 145 | return result | 164 | return result |
517 | 146 | 165 | ||
518 | 166 | @azure_ds_telemetry_reporter | ||
519 | 147 | def _get_ssh_key_from_cert(self, certificate): | 167 | def _get_ssh_key_from_cert(self, certificate): |
520 | 148 | pub_key = self._run_x509_action('-pubkey', certificate) | 168 | pub_key = self._run_x509_action('-pubkey', certificate) |
521 | 149 | keygen_cmd = ['ssh-keygen', '-i', '-m', 'PKCS8', '-f', '/dev/stdin'] | 169 | keygen_cmd = ['ssh-keygen', '-i', '-m', 'PKCS8', '-f', '/dev/stdin'] |
522 | 150 | ssh_key, _ = util.subp(keygen_cmd, data=pub_key) | 170 | ssh_key, _ = util.subp(keygen_cmd, data=pub_key) |
523 | 151 | return ssh_key | 171 | return ssh_key |
524 | 152 | 172 | ||
525 | 173 | @azure_ds_telemetry_reporter | ||
526 | 153 | def _get_fingerprint_from_cert(self, certificate): | 174 | def _get_fingerprint_from_cert(self, certificate): |
527 | 154 | """openssl x509 formats fingerprints as so: | 175 | """openssl x509 formats fingerprints as so: |
528 | 155 | 'SHA1 Fingerprint=07:3E:19:D1:4D:1C:79:92:24:C6:A0:FD:8D:DA:\ | 176 | 'SHA1 Fingerprint=07:3E:19:D1:4D:1C:79:92:24:C6:A0:FD:8D:DA:\ |
529 | @@ -163,6 +184,7 @@ class OpenSSLManager(object): | |||
530 | 163 | octets = raw_fp[eq+1:-1].split(':') | 184 | octets = raw_fp[eq+1:-1].split(':') |
531 | 164 | return ''.join(octets) | 185 | return ''.join(octets) |
532 | 165 | 186 | ||
533 | 187 | @azure_ds_telemetry_reporter | ||
534 | 166 | def _decrypt_certs_from_xml(self, certificates_xml): | 188 | def _decrypt_certs_from_xml(self, certificates_xml): |
535 | 167 | """Decrypt the certificates XML document using the our private key; | 189 | """Decrypt the certificates XML document using the our private key; |
536 | 168 | return the list of certs and private keys contained in the doc. | 190 | return the list of certs and private keys contained in the doc. |
537 | @@ -185,6 +207,7 @@ class OpenSSLManager(object): | |||
538 | 185 | shell=True, data=b'\n'.join(lines)) | 207 | shell=True, data=b'\n'.join(lines)) |
539 | 186 | return out | 208 | return out |
540 | 187 | 209 | ||
541 | 210 | @azure_ds_telemetry_reporter | ||
542 | 188 | def parse_certificates(self, certificates_xml): | 211 | def parse_certificates(self, certificates_xml): |
543 | 189 | """Given the Certificates XML document, return a dictionary of | 212 | """Given the Certificates XML document, return a dictionary of |
544 | 190 | fingerprints and associated SSH keys derived from the certs.""" | 213 | fingerprints and associated SSH keys derived from the certs.""" |
545 | @@ -265,11 +288,13 @@ class WALinuxAgentShim(object): | |||
546 | 265 | return socket.inet_ntoa(packed_bytes) | 288 | return socket.inet_ntoa(packed_bytes) |
547 | 266 | 289 | ||
548 | 267 | @staticmethod | 290 | @staticmethod |
549 | 291 | @azure_ds_telemetry_reporter | ||
550 | 268 | def _networkd_get_value_from_leases(leases_d=None): | 292 | def _networkd_get_value_from_leases(leases_d=None): |
551 | 269 | return dhcp.networkd_get_option_from_leases( | 293 | return dhcp.networkd_get_option_from_leases( |
552 | 270 | 'OPTION_245', leases_d=leases_d) | 294 | 'OPTION_245', leases_d=leases_d) |
553 | 271 | 295 | ||
554 | 272 | @staticmethod | 296 | @staticmethod |
555 | 297 | @azure_ds_telemetry_reporter | ||
556 | 273 | def _get_value_from_leases_file(fallback_lease_file): | 298 | def _get_value_from_leases_file(fallback_lease_file): |
557 | 274 | leases = [] | 299 | leases = [] |
558 | 275 | content = util.load_file(fallback_lease_file) | 300 | content = util.load_file(fallback_lease_file) |
559 | @@ -287,6 +312,7 @@ class WALinuxAgentShim(object): | |||
560 | 287 | return leases[-1] | 312 | return leases[-1] |
561 | 288 | 313 | ||
562 | 289 | @staticmethod | 314 | @staticmethod |
563 | 315 | @azure_ds_telemetry_reporter | ||
564 | 290 | def _load_dhclient_json(): | 316 | def _load_dhclient_json(): |
565 | 291 | dhcp_options = {} | 317 | dhcp_options = {} |
566 | 292 | hooks_dir = WALinuxAgentShim._get_hooks_dir() | 318 | hooks_dir = WALinuxAgentShim._get_hooks_dir() |
567 | @@ -305,6 +331,7 @@ class WALinuxAgentShim(object): | |||
568 | 305 | return dhcp_options | 331 | return dhcp_options |
569 | 306 | 332 | ||
570 | 307 | @staticmethod | 333 | @staticmethod |
571 | 334 | @azure_ds_telemetry_reporter | ||
572 | 308 | def _get_value_from_dhcpoptions(dhcp_options): | 335 | def _get_value_from_dhcpoptions(dhcp_options): |
573 | 309 | if dhcp_options is None: | 336 | if dhcp_options is None: |
574 | 310 | return None | 337 | return None |
575 | @@ -318,6 +345,7 @@ class WALinuxAgentShim(object): | |||
576 | 318 | return _value | 345 | return _value |
577 | 319 | 346 | ||
578 | 320 | @staticmethod | 347 | @staticmethod |
579 | 348 | @azure_ds_telemetry_reporter | ||
580 | 321 | def find_endpoint(fallback_lease_file=None, dhcp245=None): | 349 | def find_endpoint(fallback_lease_file=None, dhcp245=None): |
581 | 322 | value = None | 350 | value = None |
582 | 323 | if dhcp245 is not None: | 351 | if dhcp245 is not None: |
583 | @@ -352,6 +380,7 @@ class WALinuxAgentShim(object): | |||
584 | 352 | LOG.debug('Azure endpoint found at %s', endpoint_ip_address) | 380 | LOG.debug('Azure endpoint found at %s', endpoint_ip_address) |
585 | 353 | return endpoint_ip_address | 381 | return endpoint_ip_address |
586 | 354 | 382 | ||
587 | 383 | @azure_ds_telemetry_reporter | ||
588 | 355 | def register_with_azure_and_fetch_data(self, pubkey_info=None): | 384 | def register_with_azure_and_fetch_data(self, pubkey_info=None): |
589 | 356 | if self.openssl_manager is None: | 385 | if self.openssl_manager is None: |
590 | 357 | self.openssl_manager = OpenSSLManager() | 386 | self.openssl_manager = OpenSSLManager() |
591 | @@ -404,6 +433,7 @@ class WALinuxAgentShim(object): | |||
592 | 404 | 433 | ||
593 | 405 | return keys | 434 | return keys |
594 | 406 | 435 | ||
595 | 436 | @azure_ds_telemetry_reporter | ||
596 | 407 | def _report_ready(self, goal_state, http_client): | 437 | def _report_ready(self, goal_state, http_client): |
597 | 408 | LOG.debug('Reporting ready to Azure fabric.') | 438 | LOG.debug('Reporting ready to Azure fabric.') |
598 | 409 | document = self.REPORT_READY_XML_TEMPLATE.format( | 439 | document = self.REPORT_READY_XML_TEMPLATE.format( |
599 | @@ -419,6 +449,7 @@ class WALinuxAgentShim(object): | |||
600 | 419 | LOG.info('Reported ready to Azure fabric.') | 449 | LOG.info('Reported ready to Azure fabric.') |
601 | 420 | 450 | ||
602 | 421 | 451 | ||
603 | 452 | @azure_ds_telemetry_reporter | ||
604 | 422 | def get_metadata_from_fabric(fallback_lease_file=None, dhcp_opts=None, | 453 | def get_metadata_from_fabric(fallback_lease_file=None, dhcp_opts=None, |
605 | 423 | pubkey_info=None): | 454 | pubkey_info=None): |
606 | 424 | shim = WALinuxAgentShim(fallback_lease_file=fallback_lease_file, | 455 | shim = WALinuxAgentShim(fallback_lease_file=fallback_lease_file, |
LGTM