cloud-init

Merge lp:~harlowja/cloud-init/shared-wait-metadata into lp:~cloud-init-dev/cloud-init/trunk

  1. shared-wait-metadata
  2. Merge into trunk
Proposed by Joshua Harlow
Status: Rejected
Rejected by: Scott Moser
Proposed branch: lp:~harlowja/cloud-init/shared-wait-metadata
Merge into: lp:~cloud-init-dev/cloud-init/trunk
Diff against target: 231 lines (+64/-85)
3 files modified
cloudinit/sources/DataSourceEc2.py (+14/-46)
cloudinit/sources/DataSourceOpenStack.py (+13/-39)
cloudinit/util.py (+37/-0)
To merge this branch: bzr merge lp:~harlowja/cloud-init/shared-wait-metadata
Reviewer Review Type Date Requested Status
Server Team CI bot continuous-integration Needs Fixing
cloud-init Commiters Pending
Review via email: mp+206567@code.launchpad.net

Description of the change

Move shared waiting function to util

The ec2 and openstack datasources use a similar
piece of code for waiting for there metadata services
to become accessible (which varies depending on cloud,
service provider...) so its much nicer if we move that
duplicated/similar code to a standard utility method and
use that instead.

To post a comment you must log in.
Revision history for this message
Server Team CI bot (server-team-bot) wrote :

FAILED: Continuous integration, rev:951
https://server-team-jenkins.canonical.com/job/cloud-init-ci/56/
Executed test runs:
    None: https://server-team-jenkins.canonical.com/job/lp-vote-on-merge/30/console

Click here to trigger a rebuild:
https://server-team-jenkins.canonical.com/job/cloud-init-ci/56/rebuild

review: Needs Fixing (continuous-integration)
Revision history for this message
Scott Moser (smoser) wrote :

Hello,
Thank you for taking the time to contribute to cloud-init. Cloud-init has moved its revision control system to git. As a result, we are marking all bzr merge proposals as 'rejected'. If you would like to re-submit this proposal for review, please do so by following the current HACKING documentation at http://cloudinit.readthedocs.io/en/latest/topics/hacking.html .

I do think this is likely already solved at this point.

Unmerged revisions

951. By Joshua Harlow

Move shared waiting function to util

The ec2 and openstack datasources use a similar
piece of code for waiting for there metadata services
to become accessible (which varies depending on cloud,
service provider...) so its much nicer if we move that
duplicated/similar code to a standard utility method and
use that instead.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'cloudinit/sources/DataSourceEc2.py'
--- cloudinit/sources/DataSourceEc2.py 2014-02-01 20:03:32 +0000
+++ cloudinit/sources/DataSourceEc2.py 2014-02-15 00:55:16 +0000
@@ -26,7 +26,7 @@
26from cloudinit import ec2_utils as ec226from cloudinit import ec2_utils as ec2
27from cloudinit import log as logging27from cloudinit import log as logging
28from cloudinit import sources28from cloudinit import sources
29from cloudinit import url_helper as uhelp29from cloudinit import url_helper
30from cloudinit import util30from cloudinit import util
3131
32LOG = logging.getLogger(__name__)32LOG = logging.getLogger(__name__)
@@ -83,65 +83,33 @@
83 return self.metadata['instance-id']83 return self.metadata['instance-id']
8484
85 def _get_url_settings(self):85 def _get_url_settings(self):
86 mcfg = self.ds_cfg86 # max_wait < 0 indicates do not wait
87 if not mcfg:
88 mcfg = {}
89 max_wait = 12087 max_wait = 120
90 try:88 try:
91 max_wait = int(mcfg.get("max_wait", max_wait))89 max_wait = int(self.ds_cfg.get("max_wait", max_wait))
92 except Exception:90 except Exception:
93 util.logexc(LOG, "Failed to get max wait. using %s", max_wait)91 util.logexc(LOG, "Failed to get max wait. using %s", max_wait)
9492
95 timeout = 5093 timeout = 50
96 try:94 try:
97 timeout = max(0, int(mcfg.get("timeout", timeout)))95 timeout = max(0, int(self.ds_cfg.get("timeout", timeout)))
98 except Exception:96 except Exception:
99 util.logexc(LOG, "Failed to get timeout, using %s", timeout)97 util.logexc(LOG, "Failed to get timeout, using %s", timeout)
10098
101 return (max_wait, timeout)99 return (max_wait, timeout)
102100
103 def wait_for_metadata_service(self):101 def wait_for_metadata_service(self):
104 mcfg = self.ds_cfg
105 if not mcfg:
106 mcfg = {}
107
108 (max_wait, timeout) = self._get_url_settings()102 (max_wait, timeout) = self._get_url_settings()
109 if max_wait <= 0:103 md_urls = self.ds_cfg.get("metadata_urls", [])
110 return False104 md_check_path = url_helper.combine_url(self.api_ver,
111105 'meta-data', 'instance-id')
112 # Remove addresses from the list that wont resolve.106 md_url = util.wait_for_metadata_service(md_urls,
113 mdurls = mcfg.get("metadata_urls", DEF_MD_URLS)107 path=md_check_path,
114 filtered = [x for x in mdurls if util.is_resolvable_url(x)]108 max_wait=max_wait,
115109 timeout=timeout,
116 if set(filtered) != set(mdurls):110 fallback_urls=DEF_MD_URLS)
117 LOG.debug("Removed the following from metadata urls: %s",111 self.metadata_address = md_url
118 list((set(mdurls) - set(filtered))))112 return bool(md_url)
119
120 if len(filtered):
121 mdurls = filtered
122 else:
123 LOG.warn("Empty metadata url list! using default list")
124 mdurls = DEF_MD_URLS
125
126 urls = []
127 url2base = {}
128 for url in mdurls:
129 cur = "%s/%s/meta-data/instance-id" % (url, self.api_ver)
130 urls.append(cur)
131 url2base[cur] = url
132
133 start_time = time.time()
134 url = uhelp.wait_for_url(urls=urls, max_wait=max_wait,
135 timeout=timeout, status_cb=LOG.warn)
136
137 if url:
138 LOG.debug("Using metadata source: '%s'", url2base[url])
139 else:
140 LOG.critical("Giving up on md from %s after %s seconds",
141 urls, int(time.time() - start_time))
142
143 self.metadata_address = url2base.get(url)
144 return bool(url)
145113
146 def device_name_to_device(self, name):114 def device_name_to_device(self, name):
147 # Consult metadata service, that has115 # Consult metadata service, that has
148116
=== modified file 'cloudinit/sources/DataSourceOpenStack.py'
--- cloudinit/sources/DataSourceOpenStack.py 2014-02-14 19:24:06 +0000
+++ cloudinit/sources/DataSourceOpenStack.py 2014-02-15 00:55:16 +0000
@@ -16,8 +16,6 @@
16# You should have received a copy of the GNU General Public License16# You should have received a copy of the GNU General Public License
17# along with this program. If not, see <http://www.gnu.org/licenses/>.17# along with this program. If not, see <http://www.gnu.org/licenses/>.
1818
19import time
20
21from cloudinit import log as logging19from cloudinit import log as logging
22from cloudinit import sources20from cloudinit import sources
23from cloudinit import url_helper21from cloudinit import url_helper
@@ -29,6 +27,7 @@
2927
30# Various defaults/constants...28# Various defaults/constants...
31DEF_MD_URL = "http://169.254.169.254"29DEF_MD_URL = "http://169.254.169.254"
30DEF_MD_URLS = [DEF_MD_URL]
32DEFAULT_IID = "iid-dsopenstack"31DEFAULT_IID = "iid-dsopenstack"
33DEFAULT_METADATA = {32DEFAULT_METADATA = {
34 "instance-id": DEFAULT_IID,33 "instance-id": DEFAULT_IID,
@@ -54,19 +53,14 @@
54 return mstr53 return mstr
5554
56 def _get_url_settings(self):55 def _get_url_settings(self):
57 # TODO(harlowja): this is shared with ec2 datasource, we should just
58 # move it to a shared location instead...
59 # Note: the defaults here are different though.
60
61 # max_wait < 0 indicates do not wait56 # max_wait < 0 indicates do not wait
62 max_wait = -157 max_wait = -1
63 timeout = 10
64
65 try:58 try:
66 max_wait = int(self.ds_cfg.get("max_wait", max_wait))59 max_wait = int(self.ds_cfg.get("max_wait", max_wait))
67 except Exception:60 except Exception:
68 util.logexc(LOG, "Failed to get max wait. using %s", max_wait)61 util.logexc(LOG, "Failed to get max wait. using %s", max_wait)
6962
63 timeout = 10
70 try:64 try:
71 timeout = max(0, int(self.ds_cfg.get("timeout", timeout)))65 timeout = max(0, int(self.ds_cfg.get("timeout", timeout)))
72 except Exception:66 except Exception:
@@ -74,38 +68,18 @@
74 return (max_wait, timeout)68 return (max_wait, timeout)
7569
76 def wait_for_metadata_service(self):70 def wait_for_metadata_service(self):
77 urls = self.ds_cfg.get("metadata_urls", [DEF_MD_URL])
78 filtered = [x for x in urls if util.is_resolvable_url(x)]
79 if set(filtered) != set(urls):
80 LOG.debug("Removed the following from metadata urls: %s",
81 list((set(urls) - set(filtered))))
82 if len(filtered):
83 urls = filtered
84 else:
85 LOG.warn("Empty metadata url list! using default list")
86 urls = [DEF_MD_URL]
87
88 md_urls = []
89 url2base = {}
90 for url in urls:
91 md_url = url_helper.combine_url(url, 'openstack',
92 openstack.OS_LATEST,
93 'meta_data.json')
94 md_urls.append(md_url)
95 url2base[md_url] = url
96
97 (max_wait, timeout) = self._get_url_settings()71 (max_wait, timeout) = self._get_url_settings()
98 start_time = time.time()72 md_urls = self.ds_cfg.get("metadata_urls", [])
99 avail_url = url_helper.wait_for_url(urls=md_urls, max_wait=max_wait,73 md_check_path = url_helper.combine_url("openstack",
100 timeout=timeout)74 openstack.OS_LATEST,
101 if avail_url:75 'meta_data.json')
102 LOG.debug("Using metadata source: '%s'", url2base[avail_url])76 md_url = util.wait_for_metadata_service(md_urls,
103 else:77 path=md_check_path,
104 LOG.debug("Giving up on OpenStack md from %s after %s seconds",78 max_wait=max_wait,
105 md_urls, int(time.time() - start_time))79 timeout=timeout,
10680 fallback_urls=DEF_MD_URLS)
107 self.metadata_address = url2base.get(avail_url)81 self.metadata_address = md_url
108 return bool(avail_url)82 return bool(md_url)
10983
110 def get_data(self):84 def get_data(self):
111 try:85 try:
11286
=== modified file 'cloudinit/util.py'
--- cloudinit/util.py 2014-02-13 11:27:22 +0000
+++ cloudinit/util.py 2014-02-15 00:55:16 +0000
@@ -1796,6 +1796,43 @@
1796 return None1796 return None
17971797
17981798
1799def wait_for_metadata_service(urls, path=None, fallback_urls=None,
1800 max_wait=120, timeout=50):
1801 resolvable_urls = [x for x in urls if is_resolvable_url(x)]
1802 if set(resolvable_urls) != set(urls):
1803 LOG.debug("Removed the following non-resolveable metadata urls: %s",
1804 list((set(urls) - set(resolvable_urls))))
1805 if not resolvable_urls:
1806 resolvable_urls = fallback_urls
1807 if not resolvable_urls:
1808 return None
1809
1810 urls = resolvable_urls
1811 md_urls = []
1812 url2base = {}
1813 for url in urls:
1814 if path:
1815 md_url = url_helper.combine_url(url, path)
1816 else:
1817 md_url = url
1818 md_urls.append(md_url)
1819 url2base[md_url] = url
1820
1821 start_time = time.time()
1822 url = url_helper.wait_for_url(urls=md_urls, max_wait=max_wait,
1823 timeout=timeout, status_cb=LOG.warn)
1824 total_time = int(time.time() - start_time)
1825 if url:
1826 md_url = url2base[url]
1827 LOG.debug("Using metadata url: '%s' (found after %s seconds)",
1828 md_url, total_time)
1829 return md_url
1830 else:
1831 LOG.critical("Giving up on metadata from %s after %s seconds",
1832 md_urls, total_time)
1833 return None
1834
1835
1799def get_mount_info(path, log=LOG):1836def get_mount_info(path, log=LOG):
1800 # Use /proc/$$/mountinfo to find the device where path is mounted.1837 # Use /proc/$$/mountinfo to find the device where path is mounted.
1801 # This is done because with a btrfs filesystem using os.stat(path)1838 # This is done because with a btrfs filesystem using os.stat(path)