Merge lp:~harlowja/cloud-init/shared-wait-metadata into lp:~cloud-init-dev/cloud-init/trunk

[Joshua Harlow]

Move shared waiting function to util

The ec2 and openstack datasources use a similar
piece of code for waiting for there metadata services
to become accessible (which varies depending on cloud,
service provider...) so its much nicer if we move that
duplicated/similar code to a standard utility method and
use that instead.

[Server Team CI bot (server-team-bot)]

FAILED: Continuous integration, rev:951
https://server-team-jenkins.canonical.com/job/cloud-init-ci/56/
Executed test runs:
    None: https://server-team-jenkins.canonical.com/job/lp-vote-on-merge/30/console

Click here to trigger a rebuild:
https://server-team-jenkins.canonical.com/job/cloud-init-ci/56/rebuild

review: Needs Fixing (continuous-integration)

[Scott Moser (smoser)]

Hello,
Thank you for taking the time to contribute to cloud-init. Cloud-init has moved its revision control system to git. As a result, we are marking all bzr merge proposals as 'rejected'. If you would like to re-submit this proposal for review, please do so by following the current HACKING documentation at http://cloudinit.readthedocs.io/en/latest/topics/hacking.html .

I do think this is likely already solved at this point.

Unmerged revisions

951. By Joshua Harlow on 2014-02-15

Move shared waiting function to util

The ec2 and openstack datasources use a similar
piece of code for waiting for there metadata services
to become accessible (which varies depending on cloud,
service provider...) so its much nicer if we move that
duplicated/similar code to a standard utility method and
use that instead.

=== modified file 'cloudinit/sources/DataSourceEc2.py'
--- cloudinit/sources/DataSourceEc2.py	2014-02-01 20:03:32 +0000
+++ cloudinit/sources/DataSourceEc2.py	2014-02-15 00:55:16 +0000
@@ -26,7 +26,7 @@
 from cloudinit import ec2_utils as ec2
 from cloudinit import log as logging
 from cloudinit import sources
-from cloudinit import url_helper as uhelp
+from cloudinit import url_helper
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -83,65 +83,33 @@
         return self.metadata['instance-id']
 
     def _get_url_settings(self):
-        mcfg = self.ds_cfg
-        if not mcfg:
-            mcfg = {}
+        # max_wait < 0 indicates do not wait
         max_wait = 120
         try:
-            max_wait = int(mcfg.get("max_wait", max_wait))
+            max_wait = int(self.ds_cfg.get("max_wait", max_wait))
         except Exception:
             util.logexc(LOG, "Failed to get max wait. using %s", max_wait)
 
         timeout = 50
         try:
-            timeout = max(0, int(mcfg.get("timeout", timeout)))
+            timeout = max(0, int(self.ds_cfg.get("timeout", timeout)))
         except Exception:
             util.logexc(LOG, "Failed to get timeout, using %s", timeout)
 
         return (max_wait, timeout)
 
     def wait_for_metadata_service(self):
-        mcfg = self.ds_cfg
-        if not mcfg:
-            mcfg = {}
-
         (max_wait, timeout) = self._get_url_settings()
-        if max_wait <= 0:
-            return False
-
-        # Remove addresses from the list that wont resolve.
-        mdurls = mcfg.get("metadata_urls", DEF_MD_URLS)
-        filtered = [x for x in mdurls if util.is_resolvable_url(x)]
-
-        if set(filtered) != set(mdurls):
-            LOG.debug("Removed the following from metadata urls: %s",
-                      list((set(mdurls) - set(filtered))))
-
-        if len(filtered):
-            mdurls = filtered
-        else:
-            LOG.warn("Empty metadata url list! using default list")
-            mdurls = DEF_MD_URLS
-
-        urls = []
-        url2base = {}
-        for url in mdurls:
-            cur = "%s/%s/meta-data/instance-id" % (url, self.api_ver)
-            urls.append(cur)
-            url2base[cur] = url
-
-        start_time = time.time()
-        url = uhelp.wait_for_url(urls=urls, max_wait=max_wait,
-                                timeout=timeout, status_cb=LOG.warn)
-
-        if url:
-            LOG.debug("Using metadata source: '%s'", url2base[url])
-        else:
-            LOG.critical("Giving up on md from %s after %s seconds",
-                            urls, int(time.time() - start_time))
-
-        self.metadata_address = url2base.get(url)
-        return bool(url)
+        md_urls = self.ds_cfg.get("metadata_urls", [])
+        md_check_path = url_helper.combine_url(self.api_ver,
+                                               'meta-data', 'instance-id')
+        md_url = util.wait_for_metadata_service(md_urls,
+                                                path=md_check_path,
+                                                max_wait=max_wait,
+                                                timeout=timeout,
+                                                fallback_urls=DEF_MD_URLS)
+        self.metadata_address = md_url
+        return bool(md_url)
 
     def device_name_to_device(self, name):
         # Consult metadata service, that has

=== modified file 'cloudinit/sources/DataSourceOpenStack.py'
--- cloudinit/sources/DataSourceOpenStack.py	2014-02-14 19:24:06 +0000
+++ cloudinit/sources/DataSourceOpenStack.py	2014-02-15 00:55:16 +0000
@@ -16,8 +16,6 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import time
-
 from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import url_helper
@@ -29,6 +27,7 @@
 
 # Various defaults/constants...
 DEF_MD_URL = "http://169.254.169.254"
+DEF_MD_URLS = [DEF_MD_URL]
 DEFAULT_IID = "iid-dsopenstack"
 DEFAULT_METADATA = {
     "instance-id": DEFAULT_IID,
@@ -54,19 +53,14 @@
         return mstr
 
     def _get_url_settings(self):
-        # TODO(harlowja): this is shared with ec2 datasource, we should just
-        # move it to a shared location instead...
-        # Note: the defaults here are different though.
-
         # max_wait < 0 indicates do not wait
         max_wait = -1
-        timeout = 10
-
         try:
             max_wait = int(self.ds_cfg.get("max_wait", max_wait))
         except Exception:
             util.logexc(LOG, "Failed to get max wait. using %s", max_wait)
 
+        timeout = 10
         try:
             timeout = max(0, int(self.ds_cfg.get("timeout", timeout)))
         except Exception:
@@ -74,38 +68,18 @@
         return (max_wait, timeout)
 
     def wait_for_metadata_service(self):
-        urls = self.ds_cfg.get("metadata_urls", [DEF_MD_URL])
-        filtered = [x for x in urls if util.is_resolvable_url(x)]
-        if set(filtered) != set(urls):
-            LOG.debug("Removed the following from metadata urls: %s",
-                      list((set(urls) - set(filtered))))
-        if len(filtered):
-            urls = filtered
-        else:
-            LOG.warn("Empty metadata url list! using default list")
-            urls = [DEF_MD_URL]
-
-        md_urls = []
-        url2base = {}
-        for url in urls:
-            md_url = url_helper.combine_url(url, 'openstack',
-                                            openstack.OS_LATEST,
-                                            'meta_data.json')
-            md_urls.append(md_url)
-            url2base[md_url] = url
-
         (max_wait, timeout) = self._get_url_settings()
-        start_time = time.time()
-        avail_url = url_helper.wait_for_url(urls=md_urls, max_wait=max_wait,
-                                            timeout=timeout)
-        if avail_url:
-            LOG.debug("Using metadata source: '%s'", url2base[avail_url])
-        else:
-            LOG.debug("Giving up on OpenStack md from %s after %s seconds",
-                      md_urls, int(time.time() - start_time))
-
-        self.metadata_address = url2base.get(avail_url)
-        return bool(avail_url)
+        md_urls = self.ds_cfg.get("metadata_urls", [])
+        md_check_path = url_helper.combine_url("openstack",
+                                               openstack.OS_LATEST,
+                                               'meta_data.json')
+        md_url = util.wait_for_metadata_service(md_urls,
+                                                path=md_check_path,
+                                                max_wait=max_wait,
+                                                timeout=timeout,
+                                                fallback_urls=DEF_MD_URLS)
+        self.metadata_address = md_url
+        return bool(md_url)
 
     def get_data(self):
         try:

=== modified file 'cloudinit/util.py'
--- cloudinit/util.py	2014-02-13 11:27:22 +0000
+++ cloudinit/util.py	2014-02-15 00:55:16 +0000
@@ -1796,6 +1796,43 @@
     return None
 
 
+def wait_for_metadata_service(urls, path=None, fallback_urls=None,
+                              max_wait=120, timeout=50):
+    resolvable_urls = [x for x in urls if is_resolvable_url(x)]
+    if set(resolvable_urls) != set(urls):
+        LOG.debug("Removed the following non-resolveable metadata urls: %s",
+                  list((set(urls) - set(resolvable_urls))))
+    if not resolvable_urls:
+        resolvable_urls = fallback_urls
+    if not resolvable_urls:
+        return None
+
+    urls = resolvable_urls
+    md_urls = []
+    url2base = {}
+    for url in urls:
+        if path:
+            md_url = url_helper.combine_url(url, path)
+        else:
+            md_url = url
+        md_urls.append(md_url)
+        url2base[md_url] = url
+
+    start_time = time.time()
+    url = url_helper.wait_for_url(urls=md_urls, max_wait=max_wait,
+                                  timeout=timeout, status_cb=LOG.warn)
+    total_time = int(time.time() - start_time)
+    if url:
+        md_url = url2base[url]
+        LOG.debug("Using metadata url: '%s' (found after %s seconds)",
+                  md_url, total_time)
+        return md_url
+    else:
+        LOG.critical("Giving up on metadata from %s after %s seconds",
+                     md_urls, total_time)
+        return None
+
+
 def get_mount_info(path, log=LOG):
     # Use /proc/$$/mountinfo to find the device where path is mounted.
     # This is done because with a btrfs filesystem using os.stat(path)