Merge ~smoser/cloud-init:feature/login-warn into cloud-init:master

[Scott Moser]

[Scott Moser (smoser)]

There is a series of commits here that together create add support for
a.) cloud-init showing warnings to a user who ssh's in
    This is accomplished by adding a profile.d script that shows content
    in /var/lib/cloud/instance/warnings/ if it exists.

b.) warns the user when they have used a datasource that ds-identify in
    report mode did not identify.

[Ryan Harper (raharper)]

fix "create add support" -> "add support" in the above message (or at least fix in commit message).

Two comments below; generally looks good.

diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py
index 7c65257..6ff4e1c 100644
--- a/cloudinit/cmd/main.py
+++ b/cloudinit/cmd/main.py
@@ -29,6 +29,7 @@ from cloudinit import templater
 from cloudinit import url_helper
 from cloudinit import util
 from cloudinit import version
+from cloudinit import warnings
 
 from cloudinit import reporting
 from cloudinit.reporting import events
@@ -413,10 +414,48 @@ def main_init(name, args):
     # give the activated datasource a chance to adjust
     init.activate_datasource()
 
+    di_report_warn(datasource=init.datasource, cfg=init.cfg)
+
     # Stage 10
     return (init.datasource, run_module_section(mods, name, name))
 
 
+def di_report_warn(datasource, cfg):
+    if 'di_report' not in cfg:
+        LOG.debug("no di_report found in config.")
+        return
+
+    dicfg = cfg.get('di_report', {})
+    if not isinstance(dicfg, dict):
+        LOG.warn("di_report config not a dictionary: %s", dicfg)
+        return
+
+    dslist = dicfg.get('datasource_list')
+    if dslist is None:
+        LOG.warn("no 'datasource_list' found in di_report.")
+        return
+    elif not isinstance(dslist, list):
+        LOG.warn("di_report/datasource_list not a list: %s", dslist)
+        return
+
+    # ds.__module__ is like cloudinit.sources.DataSourceName
+    # where Name is the thing that shows up in datasource_list.
+    modname = datasource.__module__.rpartition(".")[2]
+    if modname.startswith(sources.DS_PREFIX):
+        modname = modname[len(sources.DS_PREFIX):]
+    else:
+        LOG.warn("Datasource '%s' came from unexpected module '%s'.",
+                 datasource, modname)
+
+    if modname in dslist:
+        LOG.debug("used datasource '%s' from '%s' was in di_report's list: %s",
+                  datasource, modname, dslist)
+        return
+
+    warnings.show_warning('dsid_missing_source', cfg,
+                          source=modname, dslist=str(dslist))
+
+
 def main_modules(action_name, args):
     name = args.mode
     # Cloud-init 'modules' stages are broken up into the following sub-stages
diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index 38f5f89..7435d58 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -340,6 +340,7 @@ class Paths(object):
             "vendordata": "vendor-data.txt.i",
             "instance_id": ".instance-id",
             "manual_clean_marker": "manual-clean",
+            "warnings": "warnings",
         }
         # Set when a datasource becomes active
         self.datasource = ds
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index c7df806..6f01a13 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -9,7 +9,6 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 import os
-import textwrap
 import time
 
 from cloudinit import ec2_utils as ec2
@@ -17,6 +16,7 @@ from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import url_helper as uhelp
 from cloudinit import util
+from cloudinit import warnings
 
 LOG = logging.getLogger(__name__)
 
@@ -224,7 +224,8 @@ class DataSourceEc2(sources.DataSource):
             return
         if self.cloud_platform == Platforms.UNKNOWN:
             warn_if_necessary(
-                util.get_cfg_by_path(cfg, STRICT_ID_PATH, STRICT_ID_DEFAULT))
+                util.get_cfg_by_path(cfg, STRICT_ID_PATH, STRICT_ID_DEFAULT),
+                cfg)
 
 
 def read_strict_mode(cfgval, default):
@@ -265,7 +266,7 @@ def parse_strict_mode(cfgval):
     return mode, sleep
 
 
-def warn_if_necessary(cfgval):
+def warn_if_necessary(cfgval, cfg):
     try:
         mode, sleep = parse_strict_mode(cfgval)
     except ValueError as e:
@@ -275,45 +276,7 @@ def warn_if_necessary(cfgval):
     if mode == "false":
         return
 
-    show_warning(sleep)
-
-
-def show_warning(sleep):
-    message = textwrap.dedent("""
-        ****************************************************************
-        # This system is using the EC2 Metadata Service, but does not  #
-        # appear to be running on Amazon EC2 or one of cloud-init's    #
-        # known platforms that provide a EC2 Metadata service. In the  #
-        # future, cloud-init may stop reading metadata from the EC2    #
-        # Metadata Service unless the platform can be identified       #
-        #                                                              #
-        # If you are seeing this message, please file a bug against    #
-        # cloud-init at https://bugs.launchpad.net/cloud-init/+filebug #
-        # Make sure to include the cloud provider your instance is     #
-        # running on.                                                  #
-        #                                                              #
-        # For more information see                                     #
-        #   https://bugs.launchpad.net/cloud-init/+bug/1660385         #
-        #                                                              #
-        # After you have filed a bug, you can disable this warning by  #
-        # launching your instance with the cloud-config below, or      #
-        # putting that content into                                    #
-        #    /etc/cloud/cloud.cfg.d/99-ec2-datasource.cfg              #
-        #                                                              #
-        # #cloud-config                                                #
-        # datasource:                                                  #
-        #  Ec2:                                                        #
-        #   strict_id: false                                           #
-        #                                                              #
-        """)
-    closemsg = ""
-    if sleep:
-        closemsg = "  [sleeping for %d seconds]  " % sleep
-    message += closemsg.center(64, "*")
-    print(message)
-    LOG.warn(message)
-    if sleep:
-        time.sleep(sleep)
+    warnings.show_warning('non_ec2_md', cfg, mode=True, sleep=sleep)
 
 
 def identify_aws(data):
diff --git a/cloudinit/warnings.py b/cloudinit/warnings.py
new file mode 100644
index 0000000..3206d4e
--- /dev/null
+++ b/cloudinit/warnings.py
@@ -0,0 +1,139 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+from cloudinit import helpers
+from cloudinit import log as logging
+from cloudinit import util
+
+import os
+import time
+
+LOG = logging.getLogger()
+
+WARNINGS = {
+    'non_ec2_md': """
+This system is using the EC2 Metadata Service, but does not appear to
+be running on Amazon EC2 or one of cloud-init's known platforms that
+provide a EC2 Metadata service. In the future, cloud-init may stop
+reading metadata from the EC2 Metadata Service unless the platform can
+be identified.
+
+If you are seeing this message, please file a bug against
+cloud-init at
+   https://bugs.launchpad.net/cloud-init/+filebug?field.tags=dsid
+Make sure to include the cloud provider your instance is
+running on.
+
+For more information see
+  https://bugs.launchpad.net/bugs/1660385
+
+After you have filed a bug, you can disable this warning by
+launching your instance with the cloud-config below, or
+putting that content into
+   /etc/cloud/cloud.cfg.d/99-ec2-datasource.cfg
+
+#cloud-config
+datasource:
+ Ec2:
+  strict_id: false""",
+    'dsid_missing_source': """
+A new feature in cloud-init identified possible datasources for
+this system as:
+  {dslist}
+However, the datasource used was: {source}
+
+In the future, cloud-init will only attempt to use datasources that
+are identified or specifically configured.
+For more information see
+  https://bugs.launchpad.net/bugs/1669675
+
+If you are seeing this message, please file a bug against
+cloud-init at
+   https://bugs.launchpad.net/cloud-init/+filebug?field.tags=dsid
+Make sure to include the cloud provider your instance is
+running on.
+
+After you have filed a bug, you can disable this warning by launching
+your instance with the cloud-config below, or putting that content
+into /etc/cloud/cloud.cfg.d/99-warnings.cfg
+
+#cloud-config
+warnings:
+  dsid_missing_source: off""",
+}
+
+
+def _get_warn_dir(cfg):
+    paths = helpers.Paths(
+        path_cfgs=cfg.get('system_info', {}).get('paths', {}))
+    return paths.get_ipath_cur('warnings')
+
+
+def _load_warn_cfg(cfg, name, mode=True, sleep=None):
+    # parse cfg['warnings']['name'] returning boolean, sleep
+    # expected value is form of:
+    #   (on|off|true|false|sleep)[,sleeptime]
+    # boolean True == on, False == off
+    default = (mode, sleep)
+    if not cfg or not isinstance(cfg, dict):
+        return default
+
+    ncfg = util.get_cfg_by_path(cfg, ('warnings', name))
+    if ncfg is None:
+        return default
+
+    if ncfg in ("on", "true", True):
+        return True, None
+
+    if ncfg in ("off", "false", False):
+        return False, None
+
+    mode, _, csleep = ncfg.partition(",")
+    if mode != "sleep":
+        return default
+
+    if csleep:
+        try:
+            sleep = int(csleep)
+        except ValueError:
+            return default
+
+    return True, sleep
+
+
+def show_warning(name, cfg=None, sleep=None, mode=True, **kwargs):
+    # kwargs are used for .format of the message.
+    # sleep and mode are default values used if
+    #   cfg['warnings']['name'] is not present.
+    if cfg is None:
+        cfg = {}
+
+    mode, sleep = _load_warn_cfg(cfg, name, mode=mode, sleep=sleep)
+    if not mode:
+        return
+
+    msg = WARNINGS[name].format(**kwargs)
+    msgwidth = 70
+    linewidth = msgwidth + 4
+
+    fmt = "# %%-%ds #" % msgwidth
+    topline = "*" * linewidth + "\n"
+    fmtlines = []
+    for line in msg.strip("\n").splitlines():
+        fmtlines.append(fmt % line)
+
+    closeline = topline
+    if sleep:
+        sleepmsg = "  [sleeping for %d seconds]  " % sleep
+        closeline = sleepmsg.center(linewidth, "*") + "\n"
+
+    util.write_file(
+        os.path.join(_get_warn_dir(cfg), name),
+        topline + "\n".join(fmtlines) + "\n" + topline)
+
+    LOG.warn(topline + "\n".join(fmtlines) + "\n" + closeline)
+
+    if sleep:
+        LOG.debug("sleeping %d seconds for warning '%s'" % (sleep, name))
+        time.sleep(sleep)
+
+# vi: ts=4 expandtab
diff --git a/packages/debian/rules.in b/packages/debian/rules.in
index 3df6053..053b764 100755
--- a/packages/debian/rules.in
+++ b/packages/debian/rules.in
@@ -12,6 +12,7 @@ override_dh_install:
 	install -d debian/cloud-init/etc/rsyslog.d
 	cp tools/21-cloudinit.conf debian/cloud-init/etc/rsyslog.d/21-cloudinit.conf
 	install -D ./tools/Z99-cloud-locale-test.sh debian/cloud-init/etc/profile.d/Z99-cloud-locale-test.sh
+	install -D ./tools/Z99-cloudinit-warnings.sh debian/cloud-init/etc/profile.d/Z99-cloudinit-warnings.sh
 
 override_dh_auto_test:
 ifeq (,$(findstring nocheck,$(DEB_BUILD_OPTIONS)))
diff --git a/tools/Z99-cloudinit-warnings.sh b/tools/Z99-cloudinit-warnings.sh
new file mode 100644
index 0000000..b237786
--- /dev/null
+++ b/tools/Z99-cloudinit-warnings.sh
@@ -0,0 +1,30 @@
+#!/bin/sh
+# This file is part of cloud-init. See LICENSE file for license information.
+
+# Purpose: show user warnings on login.
+
+cloud_init_warnings() {
+    local skipf="" warning="" idir="/var/lib/cloud/instance" n=0
+    local warndir="$idir/warnings"
+    local ufile="$HOME/.cloud-warnings.skip" sfile="$warndir/.skip"
+    [ -d "$warndir" ] || return 0
+    [ ! -f "$ufile" ] || return 0
+    [ ! -f "$skipf" ] || return 0
+
+    for warning in "$warndir"/*; do
+        [ -f "$warning" ] || continue
+        cat "$warning"
+        n=$((n+1))
+    done
+    [ $n -eq 0 ] && return 0
+    echo ""
+    echo "Disable the warnings above by:"
+    echo "  touch $ufile"
+    echo "or"
+    echo "  touch $sfile"
+}
+
+cloud_init_warnings 1>&2
+unset cloud_init_warnings
+
+# vi: syntax=sh ts=4 expandtab
diff --git a/tools/ds-identify b/tools/ds-identify
index 9711a23..d7b2a0b 100755
--- a/tools/ds-identify
+++ b/tools/ds-identify
@@ -10,8 +10,9 @@
 #   default setting is:
 #     search,found=all,maybe=all,notfound=disable
 #
-#   report: write config to /run/cloud-init/cloud.cfg.report (instead of
-#           /run/cloud-init/cloud.cfg, which effectively makes this dry-run).
+#   report: write config to /run/cloud-init/cloud.cfg, but
+#           namespaced under 'di_report'.  Thus cloud-init can still see
+#           the result, but has no affect.
 #   enable: do nothing
 #      ds-identify writes no config and just exits success
 #      the caller (cloud-init-generator) then enables cloud-init to run
@@ -108,6 +109,7 @@ DI_ON_FOUND=""
 DI_ON_MAYBE=""
 DI_ON_NOTFOUND=""
 
+DI_EC2_STRICT_ID_DEFAULT="true"
 
 error() {
     set -- "ERROR:" "$@";
@@ -720,7 +722,7 @@ dscheck_Ec2() {
         return $DS_FOUND
     fi
 
-    local default="true"
+    local default="${DI_EC2_STRICT_ID_DEFAULT}"
     if ec2_read_strict_setting "$default"; then
         strict="$_RET"
     else
@@ -867,15 +869,16 @@ _print_info() {
 }
 
 write_result() {
-    local runcfg="${PATH_RUN_CI_CFG}" ret="" line=""
-    if [ "$DI_REPORT" = "true" ]; then
-        # if report is true, then we write to .report, but touch the other.
-        : > "$runcfg"
-        runcfg="$runcfg.report"
-    fi
-    for line in "$@"; do
-        echo "$line"
-    done > "$runcfg"
+    local runcfg="${PATH_RUN_CI_CFG}" ret="" line="" pre=""
+    {
+        if [ "$DI_REPORT" = "true" ]; then
+            echo "di_report:"
+            pre="  "
+        fi
+        for line in "$@"; do
+            echo "${pre}$line";
+        done
+    } > "$runcfg"
     ret=$?
     [ $ret -eq 0 ] || {
         error "failed to write to ${runcfg}"
@@ -884,10 +887,23 @@ write_result() {
     return 0
 }
 
+record_notfound() {
+    # in report mode, report nothing was found.
+    # if not report mode: only report the negative result.
+    #   reporting an empty list would mean cloud-init would not search
+    #   any datasources.
+    if [ "$DI_REPORT" = "true" ]; then
+        found --
+    else
+        local msg="# reporting not found result. notfound=${DI_ON_NOTFOUND}."
+        local DI_REPORT="true"
+        found -- "$msg"
+    fi
+}
+
 found() {
     # found(ds1, [ds2 ...], [-- [extra lines]])
     local list="" ds=""
-    # always we write the None datasource last.
     while [ $# -ne 0 ]; do
         if [ "$1" = "--" ]; then
             shift
@@ -900,6 +916,8 @@ found() {
         # do not pass an empty line through.
         shift
     fi
+    # always write the None datasource last.
+    list="${list:+${list}, }None"
     write_result "datasource_list: [ $list ]" "$@"
     return
 }
@@ -956,6 +974,7 @@ _read_config() {
     if [ "$keyname" = "_unset" ]; then
         return 1
     fi
+    _RET=""
     return 0
 }
 
@@ -1170,13 +1189,15 @@ _main() {
         return
     fi
 
+    # record the empty result.
+    record_notfound
     case "$DI_ON_NOTFOUND" in
         $DI_DISABLED)
             debug 1 "No result. notfound=$DI_DISABLED. returning $ret_dis."
             return $ret_dis
             ;;
         $DI_ENABLED)
-            debug 1 "notfound=$DI_ENABLED. returning $ret_en"
+            debug 1 "No result. notfound=$DI_ENABLED. returning $ret_en"
             return $ret_en;;
     esac