Merge ~harlowja/cloud-init:config-entrypoints into cloud-init:master

[Joshua Harlow]

[Scott Moser (smoser)]

We discussed this MP in #cloud-init. logs available at
https://irclogs.ubuntu.com/2016/08/16/%23cloud-init.html#t18:02

I don't like entry points usage because it is so very heavy on filesystem.
Example is here http://paste.ubuntu.com/23062607/ .

Basically that just calls the suggested 'iter_entry_points' and shows the cost of doing so.

It is *very* heavy to do this, and i'm dis-interested right now in slowing down cloud-init.

review: Disapprove

Unmerged commits

8ad0b3d... by Joshua Harlow on 2016-08-10

Use entrypoints for cloudinit.config

Instead of looking in a very specific location for
cloudinit config modules; which for those adding there
own modules makes it hard to do without patching that
location instead use entrypoints and register all
current cloudinit config modules by default with that
new entrypoint (and use that same entrypoint namespace
for later finding needed modules).

diff --git a/cloudinit/config/__init__.py b/cloudinit/config/__init__.py
index d57453b..948ce5f 100644
--- a/cloudinit/config/__init__.py
+++ b/cloudinit/config/__init__.py
@@ -39,8 +39,6 @@ def form_module_name(name):
     canon_name = canon_name.strip()
     if not canon_name:
         return None
-    if not canon_name.startswith(MOD_PREFIX):
-        canon_name = '%s%s' % (MOD_PREFIX, canon_name)
     return canon_name
 
 
diff --git a/cloudinit/importer.py b/cloudinit/importer.py
index fb57253..fb68d99 100644
--- a/cloudinit/importer.py
+++ b/cloudinit/importer.py
@@ -20,6 +20,7 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import pkg_resources
 import sys
 
 
@@ -56,3 +57,13 @@ def find_module(base_name, search_paths, required_attrs=None):
         if found_attrs == len(required_attrs):
             found_paths.append(full_path)
     return (found_paths, lookup_paths)
+
+
+def iter_entry_points(namespace, loader=None, name=None):
+    for e in pkg_resources.iter_entry_points(namespace, name=name):
+        if loader is None:
+            thing = e.load()
+        else:
+            thing = loader(e)
+        if thing is not None:
+            yield (e.name, thing)
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 47deac6..beb3cba 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -54,6 +54,7 @@ LOG = logging.getLogger(__name__)
 
 NULL_DATA_SOURCE = None
 NO_PREVIOUS_INSTANCE_ID = "NO_PREVIOUS_INSTANCE_ID"
+EP_CONFIG_NAMESPACE = 'cloudinit.config'
 
 
 class Init(object):
@@ -731,21 +732,39 @@ class Modules(object):
             raw_name = raw_mod['mod']
             freq = raw_mod.get('freq')
             run_args = raw_mod.get('args') or []
-            mod_name = config.form_module_name(raw_name)
-            if not mod_name:
+            canon_name = config.form_module_name(raw_name)
+            if not canon_name:
                 continue
+            ep_locs = list(
+                importer.iter_entry_points(EP_CONFIG_NAMESPACE,
+                                           name=canon_name))
+            if not ep_locs:
+                LOG.warn("Could not find entrypoint matching name '%s'"
+                         " (which is the canonicalized form of '%s')"
+                         " registered under namespace '%s'",
+                         canon_name, raw_name, EP_CONFIG_NAMESPACE)
+                continue
+            ep_names = [mod_name for (mod_name, _mod) in ep_locs]
+            if len(ep_names) > 1:
+                LOG.warn("Selecting '%s' entrypoint for configuration"
+                         " module '%s' (which is the canonicalized form"
+                         " of '%s') even though %s other entrypoints were"
+                         " found", ep_names[0], canon_name, raw_name,
+                         ep_names)
+            else:
+                LOG.debug("Selecting '%s' entrypoint for configuration"
+                          " module '%s' (which is the canonicalized"
+                          " form of '%s')", ep_names[0], canon_name, raw_name)
+            # Entry points are yielded from the active distributions in
+            # the order that the distributions appear on sys.path
+            ep_name, mod = ep_locs[0]
+            mod = config.fixup_module(mod)
             if freq and freq not in FREQUENCIES:
-                LOG.warn(("Config specified module %s"
-                          " has an unknown frequency %s"), raw_name, freq)
+                LOG.warn("Config specified module '%s' (found at entrypoint"
+                         " '%s') has an unknown frequency %s",
+                         raw_name, ep_name, freq)
                 # Reset it so when ran it will get set to a known value
                 freq = None
-            mod_locs, looked_locs = importer.find_module(
-                mod_name, ['', type_utils.obj_name(config)], ['handle'])
-            if not mod_locs:
-                LOG.warn("Could not find module named %s (searched %s)",
-                         mod_name, looked_locs)
-                continue
-            mod = config.fixup_module(importer.import_module(mod_locs[0]))
             mostly_mods.append([mod, raw_name, freq, run_args])
         return mostly_mods
 
diff --git a/setup.py b/setup.py
index 4abbb67..1949dd1 100755
--- a/setup.py
+++ b/setup.py
@@ -196,6 +196,21 @@ requirements = read_requires()
 if sys.version_info < (3,):
     requirements.append('cheetah')
 
+config_eps = []
+for filename in os.listdir(os.path.join(os.getcwd(), "cloudinit", "config")):
+    if filename.startswith("_"):
+        continue
+    else:
+        if filename.endswith(".py"):
+            realname = filename[0:-3]
+            if realname.startswith("cc_"):
+                basename = realname[3:]
+            else:
+                basename = realname
+            config_eps.append(
+                "%s = cloudinit.config.%s" % (basename, realname))
+
+
 setuptools.setup(
     name='cloud-init',
     version=get_version(),
@@ -213,5 +228,6 @@ setuptools.setup(
         'console_scripts': [
             'cloud-init = cloudinit.cmd.main:main'
         ],
+        'cloudinit.config': config_eps,
     }
 )