Merge ~smoser/cloud-init:feature/ds-identify-warn into cloud-init:master

Proposed by Scott Moser
Status: Merged
Merged at revision: 83606aecaae571ce8eb7d6499de028192d82f79b
Proposed branch: ~smoser/cloud-init:feature/ds-identify-warn
Merge into: cloud-init:master
Diff against target: 597 lines (+365/-41)
3 files modified
cloudinit/sources/DataSourceAliYun.py (+4/-0)
cloudinit/sources/DataSourceEc2.py (+177/-2)
tools/ds-identify (+184/-39)
Reviewer Review Type Date Requested Status
Server Team CI bot continuous-integration Needs Fixing
cloud-init Commiters Pending
Review via email: mp+318282@code.launchpad.net
To post a comment you must log in.
Revision history for this message
Server Team CI bot (server-team-bot) wrote :

FAILED: Continuous integration, rev:ded240b6a6859e12d3b25d86fbfb4ade3d7d2d44
No commit message was specified in the merge proposal. Click on the following link and set the commit message (if you want a jenkins rebuild you need to trigger it yourself):
https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/318282/+edit-commit-message

https://jenkins.ubuntu.com/server/job/cloud-init-ci/68/
Executed test runs:
    SUCCESS: https://jenkins.ubuntu.com/server/job/cloud-init-ci/nodes=metal-arm64/68
    SUCCESS: https://jenkins.ubuntu.com/server/job/cloud-init-ci/nodes=metal-ppc64el/68
    SUCCESS: https://jenkins.ubuntu.com/server/job/cloud-init-ci/nodes=metal-s390x/68
    SUCCESS: https://jenkins.ubuntu.com/server/job/cloud-init-ci/nodes=vm-amd64/68
    SUCCESS: https://jenkins.ubuntu.com/server/job/cloud-init-ci/nodes=vm-i386/68

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/68/rebuild

review: Needs Fixing (continuous-integration)

There was an error fetching revisions from git servers. Please try again in a few minutes. If the problem persists, contact Launchpad support.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/cloudinit/sources/DataSourceAliYun.py b/cloudinit/sources/DataSourceAliYun.py
2index 2d00255..9debe94 100644
3--- a/cloudinit/sources/DataSourceAliYun.py
4+++ b/cloudinit/sources/DataSourceAliYun.py
5@@ -22,6 +22,10 @@ class DataSourceAliYun(EC2.DataSourceEc2):
6 def get_public_ssh_keys(self):
7 return parse_public_keys(self.metadata.get('public-keys', {}))
8
9+ @property
10+ def cloud_platform(self):
11+ return EC2.Platforms.ALIYUN
12+
13
14 def parse_public_keys(public_keys):
15 keys = []
16diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
17index c657fd0..898694b 100644
18--- a/cloudinit/sources/DataSourceEc2.py
19+++ b/cloudinit/sources/DataSourceEc2.py
20@@ -9,6 +9,7 @@
21 # This file is part of cloud-init. See LICENSE file for license information.
22
23 import os
24+import textwrap
25 import time
26
27 from cloudinit import ec2_utils as ec2
28@@ -22,12 +23,24 @@ LOG = logging.getLogger(__name__)
29 # Which version we are requesting of the ec2 metadata apis
30 DEF_MD_VERSION = '2009-04-04'
31
32+STRICT_ID_PATH = ("datasource", "Ec2", "strict_id")
33+STRICT_ID_DEFAULT = "warn"
34+
35+
36+class Platforms(object):
37+ ALIYUN = "AliYun"
38+ AWS = "AWS"
39+ BRIGHTBOX = "Brightbox"
40+ SEEDED = "Seeded"
41+ UNKNOWN = "Unknown"
42+
43
44 class DataSourceEc2(sources.DataSource):
45 # Default metadata urls that will be used if none are provided
46 # They will be checked for 'resolveability' and some of the
47 # following may be discarded if they do not resolve
48 metadata_urls = ["http://169.254.169.254", "http://instance-data.:8773"]
49+ _cloud_platform = None
50
51 def __init__(self, sys_cfg, distro, paths):
52 sources.DataSource.__init__(self, sys_cfg, distro, paths)
53@@ -41,8 +54,18 @@ class DataSourceEc2(sources.DataSource):
54 self.userdata_raw = seed_ret['user-data']
55 self.metadata = seed_ret['meta-data']
56 LOG.debug("Using seeded ec2 data from %s", self.seed_dir)
57+ self._cloud_platform = Platforms.SEEDED
58 return True
59
60+ strict_mode, _sleep = read_strict_mode(
61+ util.get_cfg_by_path(self.sys_cfg, STRICT_ID_PATH,
62+ STRICT_ID_DEFAULT), ("warn", None))
63+
64+ LOG.debug("strict_mode: %s, cloud_platform=%s",
65+ strict_mode, self.cloud_platform)
66+ if strict_mode == "true" and self.cloud_platform == Platforms.UNKNOWN:
67+ return False
68+
69 try:
70 if not self.wait_for_metadata_service():
71 return False
72@@ -51,8 +74,8 @@ class DataSourceEc2(sources.DataSource):
73 ec2.get_instance_userdata(self.api_ver, self.metadata_address)
74 self.metadata = ec2.get_instance_metadata(self.api_ver,
75 self.metadata_address)
76- LOG.debug("Crawl of metadata service took %s seconds",
77- int(time.time() - start_time))
78+ LOG.debug("Crawl of metadata service took %.3f seconds",
79+ time.time() - start_time)
80 return True
81 except Exception:
82 util.logexc(LOG, "Failed reading from metadata address %s",
83@@ -190,6 +213,158 @@ class DataSourceEc2(sources.DataSource):
84 return az[:-1]
85 return None
86
87+ @property
88+ def cloud_platform(self):
89+ if self._cloud_platform is None:
90+ self._cloud_platform = identify_platform()
91+ return self._cloud_platform
92+
93+ def activate(self, cfg, is_new_instance):
94+ if not is_new_instance:
95+ return
96+ if self.cloud_platform == Platforms.UNKNOWN:
97+ warn_if_necessary(
98+ util.get_cfg_by_path(cfg, STRICT_ID_PATH, STRICT_ID_DEFAULT))
99+
100+
101+def read_strict_mode(cfgval, default):
102+ try:
103+ return parse_strict_mode(cfgval)
104+ except ValueError as e:
105+ LOG.warn(e)
106+ return default
107+
108+
109+def parse_strict_mode(cfgval):
110+ # given a mode like:
111+ # true, false, warn,[sleep]
112+ # return tuple with string mode (true|false|warn) and sleep.
113+ if cfgval is True:
114+ return 'true', None
115+ if cfgval is False:
116+ return 'false', None
117+
118+ if not cfgval:
119+ return 'warn', 0
120+
121+ mode, _, sleep = cfgval.partition(",")
122+ if mode not in ('true', 'false', 'warn'):
123+ raise ValueError(
124+ "Invalid mode '%s' in strict_id setting '%s': "
125+ "Expected one of 'true', 'false', 'warn'." % (mode, cfgval))
126+
127+ if sleep:
128+ try:
129+ sleep = int(sleep)
130+ except ValueError:
131+ raise ValueError("Invalid sleep '%s' in strict_id setting '%s': "
132+ "not an integer" % (sleep, cfgval))
133+ else:
134+ sleep = None
135+
136+ return mode, sleep
137+
138+
139+def warn_if_necessary(cfgval):
140+ try:
141+ mode, sleep = parse_strict_mode(cfgval)
142+ except ValueError as e:
143+ LOG.warn(e)
144+ return
145+
146+ if mode == "false":
147+ return
148+
149+ show_warning()
150+ if sleep:
151+ time.sleep(sleep)
152+
153+
154+def show_warning():
155+ message = textwrap.dedent("""
156+ # **************************************************************
157+ # This system is using the EC2 Metadata Service, but does not #
158+ # appear to be running on Amazon EC2. In the future, #
159+ # cloud-init may stop reading metadata from the EC2 Metadata #
160+ # Service unless the platform can be identified locally. #
161+ # #
162+ # If you are seeing this message, please file a bug against #
163+ # cloud-init at https://bugs.launchpad.net/cloud-init/+filebug #
164+ # Make sure to include the cloud provider your instance is #
165+ # running on. #
166+ # #
167+ # For more information see #
168+ # https://bugs.launchpad.net/cloud-init/+bug/1660385 #
169+ # #
170+ # After you have filed a bug, you can disable this warning by #
171+ # launching your instance with the cloud-config below, or #
172+ # putting that content into #
173+ # /etc/cloud/cloud.cfg.d/99-ec2-datasource.cfg #
174+ # #
175+ # #cloud-config #
176+ # datasource: #
177+ # Ec2: #
178+ # strict_id: false #
179+ # #
180+ ****************************************************************""")
181+ print(message)
182+ LOG.warn(message)
183+
184+
185+def identify_aws(data):
186+ # data is a dictionary returned by _collect_platform_data.
187+ if (data['uuid'].startswith('ec2') and
188+ (data['uuid_source'] == 'hypervisor' or
189+ data['uuid'] == data['serial'])):
190+ return Platforms.AWS
191+
192+ return None
193+
194+
195+def identify_brightbox(data):
196+ if data['serial'].endswith('brightbox.com'):
197+ return Platforms.BRIGHTBOX
198+
199+
200+def identify_platform():
201+ # identify the platform and return an entry in Platforms.
202+ data = _collect_platform_data()
203+ checks = (identify_aws, identify_brightbox, lambda x: Platforms.UNKNOWN)
204+ for checker in checks:
205+ try:
206+ result = checker(data)
207+ if result:
208+ return result
209+ except Exception as e:
210+ LOG.warn("calling %s with %s raised exception: %s",
211+ checker, data, e)
212+
213+
214+def _collect_platform_data():
215+ # returns a dictionary with all lower case values:
216+ # uuid: system-uuid from dmi or /sys/hypervisor
217+ # uuid_source: 'hypervisor' (/sys/hypervisor/uuid) or 'dmi'
218+ # serial: dmi 'system-serial-number' (/sys/.../product_serial)
219+ data = {}
220+ try:
221+ uuid = util.load_file("/sys/hypervisor/uuid").strip()
222+ data['uuid_source'] = 'hypervisor'
223+ except Exception:
224+ uuid = util.read_dmi_data('system-uuid')
225+ data['uuid_source'] = 'dmi'
226+
227+ if uuid is None:
228+ uuid = ''
229+ data['uuid'] = uuid.lower()
230+
231+ serial = util.read_dmi_data('system-serial-number')
232+ if serial is None:
233+ serial = ''
234+
235+ data['serial'] = serial.lower()
236+
237+ return data
238+
239
240 # Used to match classes to dependencies
241 datasources = [
242diff --git a/tools/ds-identify b/tools/ds-identify
243index 7bb6386..34bf064 100755
244--- a/tools/ds-identify
245+++ b/tools/ds-identify
246@@ -4,12 +4,12 @@
247 # or on the kernel command line. It takes primarily 2 inputs:
248 # datasource: can specify the datasource that should be used.
249 # kernel command line option: ci.datasource=<dsname>
250-#
251+#
252 # policy: a string that indicates how ds-identify should operate.
253 # kernel command line option: ci.di.policy=<policy>
254 # default setting is:
255 # search,found=all,maybe=all,notfound=disable
256-
257+#
258 # report: write config to /run/cloud-init/cloud.cfg.report (instead of
259 # /run/cloud-init/cloud.cfg, which effectively makes this dry-run).
260 # enable: do nothing
261@@ -29,17 +29,15 @@
262 # all: enable all DS_MAYBE
263 # none: ignore any DS_MAYBE
264 #
265-# notfound: (default=disable)
266-# disable: disable cloud-init
267-# enable: enable cloud-init
268+# notfound: (default=disabled)
269+# disabled: disable cloud-init
270+# enabled: enable cloud-init
271 #
272+# ci.datasource.ec2.strict_id: (true|false|warn[,0-9])
273+# if ec2 datasource does not strictly match,
274+# return not_found if true
275+# return maybe if false or warn*.
276 #
277-# zesty:
278-# policy: found=first,maybe=all,none=disable
279-# xenial:
280-# policy: found=all,maybe=all,none=enable
281-# and then at a later date
282-
283
284 set -u
285 set -f
286@@ -561,10 +559,45 @@ dscheck_OpenNebula() {
287 return ${DS_NOT_FOUND}
288 }
289
290+ovf_vmware_guest_customization() {
291+ # vmware guest customization
292+
293+ # virt provider must be vmware
294+ [ "${DI_VIRT}" = "vmware" ] || return 1
295+
296+ # we have to have the plugin to do vmware customization
297+ local found="" pkg="" pre="/usr/lib"
298+ for pkg in vmware-tools open-vm-tools; do
299+ if [ -f "$pre/$pkg/plugins/vmsvc/libdeployPkgPlugin.so" ]; then
300+ found="$pkg"; break;
301+ fi
302+ done
303+ [ -n "$found" ] || return 1
304+
305+ # disable_vmware_customization defaults to False.
306+ # any value then other than false means disabled.
307+ local key="disable_vmware_customization"
308+ local match="" bp="${PATH_CLOUD_CONFD}/cloud.cfg"
309+ match="$bp.d/*[Oo][Vv][Ff]*.cfg"
310+ if check_config "$key" "$match"; then
311+ debug 2 "${_RET_fname} set $key to $_RET"
312+ case "$_RET" in
313+ 0|false|False) return 0;;
314+ *) return;;
315+ esac
316+ fi
317+
318+ return 1
319+}
320+
321 dscheck_OVF() {
322 local p=""
323 check_seed_dir ovf ovf-env.xml && return "${DS_FOUND}"
324
325+ if ovf_vmware_guest_customization; then
326+ return ${DS_FOUND}
327+ fi
328+
329 has_cdrom || return ${DS_NOT_FOUND}
330
331 # FIXME: currently just return maybe if there is a cdrom
332@@ -595,36 +628,119 @@ dscheck_Bigstep() {
333 return ${DS_NOT_FOUND}
334 }
335
336-dscheck_Ec2() {
337- # http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/identify_ec2_instances.html
338- # http://paste.ubuntu.com/23630859/
339+ec2_read_strict_setting() {
340+ # the 'strict_id' setting for Ec2 controls behavior when
341+ # the platform does not identify itself directly as Ec2.
342+ # order of precedence is:
343+ # 1. builtin setting here cloud-init/ds-identify builtin
344+ # 2. ds-identify config
345+ # 3. system config (/etc/cloud/cloud.cfg.d/*Ec2*.cfg)
346+ # 4. kernel command line (undocumented)
347+ # 5. user-data or vendor-data (not available here)
348+ local default="$1" key="ci.datasource.ec2.strict_id" val=""
349+
350+ # 4. kernel command line
351+ case " ${DI_KERNEL_CMDLINE} " in
352+ *\ $key=*\ )
353+ val=${DI_KERNEL_CMDLINE##*$key=}
354+ val=${val%% *};
355+ _RET=${val:-$default}
356+ return 0
357+ esac
358+
359+ # 3. look for the key 'strict_id' (datasource/Ec2/strict_id)
360+ local match="" bp="${PATH_CLOUD_CONFD}/cloud.cfg"
361+ match="$bp.d/*[Ee][Cc]2*.cfg"
362+ if check_config strict_id "$match"; then
363+ debug 2 "${_RET_fname} set strict_id to $_RET"
364+ return 0
365+ fi
366+
367+ # 2. ds-identify config (datasource.ec2.strict)
368+ local config="${PATH_DI_CONFIG}"
369+ if [ -f "$config" ]; then
370+ if _read_config "$key" < "$config"; then
371+ _RET=${_RET:-$default}
372+ return 0
373+ fi
374+ fi
375+
376+ # 1. Default
377+ _RET=$default
378+ return 0
379+}
380+
381+ec2_identify_platform() {
382+ local default="$1"
383+ local serial="${DI_DMI_PRODUCT_SERIAL}"
384+
385+ # brightbox https://bugs.launchpad.net/cloud-init/+bug/1661693
386+ case "$serial" in
387+ *brightbox.com) _RET="Brightbox"; return 0;;
388+ esac
389+
390+ # AWS http://docs.aws.amazon.com/AWSEC2/
391+ # latest/UserGuide/identify_ec2_instances.html
392 local uuid="" hvuuid="$PATH_ROOT/sys/hypervisor/uuid"
393- is_container && return ${DS_NOT_FOUND}
394 # if the (basically) xen specific /sys/hypervisor/uuid starts with 'ec2'
395 if [ -r "$hvuuid" ] && read uuid < "$hvuuid" &&
396 [ "${uuid#ec2}" != "$uuid" ]; then
397- return ${DS_FOUND}
398+ _RET="AWS"
399+ return 0
400 fi
401
402 # product uuid and product serial start with case insensitive
403- local uuid=${DI_DMI_PRODUCT_UUID} serial=${DI_DMI_PRODUCT_SERIAL}
404+ local uuid="${DI_DMI_PRODUCT_UUID}"
405 case "$uuid:$serial" in
406 [Ee][Cc]2*:[Ee][Cc]2)
407 # both start with ec2, now check for case insenstive equal
408- nocase_equal "$uuid" "$serial" && return ${DS_FOUND};;
409+ nocase_equal "$uuid" "$serial" &&
410+ { _RET="AWS"; return 0; };;
411 esac
412
413- # search through config files to check for platform
414- local f="" match="${PATH_CLOUD_CONFD}/*ec2*.cfg"
415- # look for the key 'platform' (datasource/ec2/look_alike/behavior)
416- if check_config platform "$match"; then
417- if [ "$platform" != "Unknown" ]; then
418- _RET="$name"
419- return "${DS_FOUND}"
420- fi
421+ _RET="$default"
422+ return 0;
423+}
424+
425+dscheck_Ec2() {
426+ check_seed_dir "ec2" meta-data user-data && return ${DS_FOUND}
427+ is_container && return ${DS_NOT_FOUND}
428+
429+ local unknown="Unknown" platform=""
430+ if ec2_identify_platform "$unknown"; then
431+ platform="$_RET"
432+ else
433+ warn "Failed to identify ec2 platform. Using '$unknown'."
434+ platform=$unknown
435 fi
436
437- return ${DS_NOT_FOUND}
438+ debug 1 "ec2 platform is '$platform'."
439+ if [ "$platform" != "$unknown" ]; then
440+ return $DS_FOUND
441+ fi
442+
443+ local default="true"
444+ if ec2_read_strict_setting "$default"; then
445+ strict="$_RET"
446+ else
447+ debug 1 "ec2_read_strict returned non-zero: $?. using '$default'."
448+ strict="$default"
449+ fi
450+
451+ local key="datasource/Ec2/strict_id"
452+ case "$strict" in
453+ true|false|warn|warn,[0-9]*) :;;
454+ *)
455+ warn "$key was set to invalid '$strict'. using '$default'"
456+ strict="$default";;
457+ esac
458+
459+ _RET_excfg="datasource: {Ec2: {strict_id: \"$strict\"}}"
460+ if [ "$strict" = "true" ]; then
461+ return $DS_NOT_FOUND
462+ else
463+ return $DS_MAYBE
464+ fi
465 }
466
467 dscheck_GCE() {
468@@ -768,12 +884,22 @@ write_result() {
469 }
470
471 found() {
472+ # found(ds1, [ds2 ...], [-- [extra lines]])
473 local list="" ds=""
474 # always we write the None datasource last.
475- for ds in "$@" None; do
476- list="${list:+${list}, }$ds"
477+ while [ $# -ne 0 ]; do
478+ if [ "$1" = "--" ]; then
479+ shift
480+ break
481+ fi
482+ list="${list:+${list}, }$1"
483+ shift
484 done
485- write_result "datasource_list: [ $list ]"
486+ if [ $# -eq 1 ] && [ -z "$1" ]; then
487+ # do not pass an empty line through.
488+ shift
489+ fi
490+ write_result "datasource_list: [ $list ]" "$@"
491 return
492 }
493
494@@ -794,8 +920,10 @@ unquote() {
495 }
496
497 _read_config() {
498- # reads config from stdin, modifies _rc scoped environment vars.
499- # rc_policy and _rc_dsname
500+ # reads config from stdin,
501+ # if no parameters are set, modifies _rc scoped environment vars.
502+ # if keyname is provided, then returns found value of that key.
503+ local keyname="${1:-_unset}"
504 local line="" hash="#" ckey="" key="" val=""
505 while read line; do
506 line=${line%%${hash}*}
507@@ -806,15 +934,28 @@ _read_config() {
508 trim "$key"
509 key=${_RET}
510
511+ [ "$keyname" != "_unset" ] && [ "$keyname" != "$key" ] &&
512+ continue
513+
514 val="${line#*:}"
515 trim "$val"
516 unquote "${_RET}"
517 val=${_RET}
518+
519+ if [ "$keyname" = "$key" ]; then
520+ _RET="$val"
521+ return 0
522+ fi
523+
524 case "$key" in
525 datasource) _rc_dsname="$val";;
526 policy) _rc_policy="$val";;
527 esac
528 done
529+ if [ "$keyname" = "_unset" ]; then
530+ return 1
531+ fi
532+ return 0
533 }
534
535 parse_warn() {
536@@ -889,7 +1030,7 @@ parse_policy() {
537 }
538
539 read_config() {
540- local config=${PATH_DI_CONFIG}
541+ local config="${PATH_DI_CONFIG}"
542 local _rc_dsname="" _rc_policy="" ret=""
543 if [ -f "$config" ]; then
544 _read_config < "$config"
545@@ -980,7 +1121,8 @@ _main() {
546 return
547 fi
548
549- local found="" ret="" ds="" maybe=""
550+ local found="" ret="" ds="" maybe="" _RET_excfg=""
551+ local exfound_cfg="" exmaybe_cfg=""
552 for ds in ${DI_DSLIST}; do
553 dscheck_fn="dscheck_${ds}"
554 debug 2 "Checking for datasource '$ds' via '$dscheck_fn'"
555@@ -988,20 +1130,23 @@ _main() {
556 warn "No check method '$dscheck_fn' for datasource '$ds'"
557 continue
558 fi
559+ _RET_excfg=""
560 $dscheck_fn
561 ret="$?"
562 case "$ret" in
563 $DS_FOUND)
564 debug 1 "check for '$ds' returned found";
565+ exfound_cfg="${exfound_cfg:+${exfound_cfg}${CR}}${_RET_excfg}"
566 found="${found} $ds";;
567 $DS_MAYBE)
568- debug 1 "check for $ds returned maybe";
569+ debug 1 "check for '$ds' returned maybe";
570+ exmaybe_cfg="${exmaybe_cfg:+${exmaybe_cfg}${CR}}${_RET_excfg}"
571 maybe="${maybe} $ds";;
572- *) debug 2 "check for $ds returned not-found[$ret]";;
573+ *) debug 2 "check for '$ds' returned not-found[$ret]";;
574 esac
575 done
576
577- debug 2 "found=$found maybe=$maybe"
578+ debug 2 "found=${found# } maybe=${maybe# }"
579 set -- $found
580 if [ $# -ne 0 ]; then
581 if [ $# -eq 1 ]; then
582@@ -1013,14 +1158,14 @@ _main() {
583 set -- "$1"
584 fi
585 fi
586- found "$@"
587+ found "$@" -- "${exfound_cfg}"
588 return
589 fi
590
591 set -- $maybe
592 if [ $# -ne 0 -a "${DI_ON_MAYBE}" != "none" ]; then
593 debug 1 "$# datasources returned maybe: $*"
594- found "$@"
595+ found "$@" -- "${exmaybe_cfg}"
596 return
597 fi
598

Subscribers

People subscribed via source and target branches