Merge lp:~smoser/cloud-init/mirror-rework into lp:~cloud-init-dev/cloud-init/trunk
- mirror-rework
- Merge into trunk
Status: | Merged |
---|---|
Merged at revision: | 630 |
Proposed branch: | lp:~smoser/cloud-init/mirror-rework |
Merge into: | lp:~cloud-init-dev/cloud-init/trunk |
Diff against target: |
591 lines (+309/-108) 11 files modified
ChangeLog (+4/-0) cloudinit/cloud.py (+0/-3) cloudinit/config/cc_apt_update_upgrade.py (+74/-57) cloudinit/distros/__init__.py (+71/-2) cloudinit/distros/debian.py (+4/-0) cloudinit/sources/DataSourceCloudStack.py (+2/-1) cloudinit/sources/DataSourceEc2.py (+6/-34) cloudinit/sources/__init__.py (+7/-3) config/cloud.cfg (+14/-2) templates/sources.list.tmpl (+6/-6) tests/unittests/test_distros/test_generic.py (+121/-0) |
To merge this branch: | bzr merge lp:~smoser/cloud-init/mirror-rework |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
cloud-init Commiters | Pending | ||
Review via email: mp+120852@code.launchpad.net |
Commit message
rework package mirror selection
There are several changes here.
* Datasource now has a 'availability_zone' getter.
* get_package_
* Datasource convenience 'get_package_
the configured distro, and passes it the availability-zone
* distro has a get_package_
* get_package_
this is to facilitate use of 'security' and 'primary' archive.
* this supports searching based on templates. Any template
that references undefined values is skipped. These templates
can contain 'availability_zone' (LP: #1037727)
* distro's mirrors can be arch specific (LP: #1028501)
* rename_apt_lists supports the "mirror_info" rather than single mirror
* generate_
ubuntu mirrors reference '$security' rather than security (LP: #1006963)
* remove the DataSourceEc2 specific mirror selection, but instead
rely on the above filtering, and the fact that 'ec2_region' is only
defined if the availability_zone looks like a ec2 az.
Description of the change
- 634. By Scott Moser
-
improve the check for "uses unknown key" in mirror templates
instead of substituting and then checking for presense of a unlikely to
occur string, this only adds to the search list if there is no KeyError
raised.
Preview Diff
1 | === modified file 'ChangeLog' | |||
2 | --- ChangeLog 2012-08-21 01:27:04 +0000 | |||
3 | +++ ChangeLog 2012-08-22 19:21:17 +0000 | |||
4 | @@ -1,4 +1,8 @@ | |||
5 | 1 | 0.7.0: | 1 | 0.7.0: |
6 | 2 | - allow distro mirror selection to include availability-zone (LP: #1037727) | ||
7 | 3 | - allow arch specific mirror selection (select ports.ubuntu.com on arm) | ||
8 | 4 | LP: #1028501 | ||
9 | 5 | - allow specification of security mirrors (LP: #1006963) | ||
10 | 2 | - add the 'None' datasource (LP: #906669), which will allow jobs | 6 | - add the 'None' datasource (LP: #906669), which will allow jobs |
11 | 3 | to run even if there is no "real" datasource found. | 7 | to run even if there is no "real" datasource found. |
12 | 4 | - write ssh authorized keys to console, ssh_authkey_fingerprints | 8 | - write ssh authorized keys to console, ssh_authkey_fingerprints |
13 | 5 | 9 | ||
14 | === modified file 'cloudinit/cloud.py' | |||
15 | --- cloudinit/cloud.py 2012-06-21 16:12:16 +0000 | |||
16 | +++ cloudinit/cloud.py 2012-08-22 19:21:17 +0000 | |||
17 | @@ -82,9 +82,6 @@ | |||
18 | 82 | def get_locale(self): | 82 | def get_locale(self): |
19 | 83 | return self.datasource.get_locale() | 83 | return self.datasource.get_locale() |
20 | 84 | 84 | ||
21 | 85 | def get_local_mirror(self): | ||
22 | 86 | return self.datasource.get_local_mirror() | ||
23 | 87 | |||
24 | 88 | def get_hostname(self, fqdn=False): | 85 | def get_hostname(self, fqdn=False): |
25 | 89 | return self.datasource.get_hostname(fqdn=fqdn) | 86 | return self.datasource.get_hostname(fqdn=fqdn) |
26 | 90 | 87 | ||
27 | 91 | 88 | ||
28 | === modified file 'cloudinit/config/cc_apt_update_upgrade.py' | |||
29 | --- cloudinit/config/cc_apt_update_upgrade.py 2012-08-22 18:12:32 +0000 | |||
30 | +++ cloudinit/config/cc_apt_update_upgrade.py 2012-08-22 19:21:17 +0000 | |||
31 | @@ -50,20 +50,25 @@ | |||
32 | 50 | upgrade = util.get_cfg_option_bool(cfg, 'apt_upgrade', False) | 50 | upgrade = util.get_cfg_option_bool(cfg, 'apt_upgrade', False) |
33 | 51 | 51 | ||
34 | 52 | release = get_release() | 52 | release = get_release() |
37 | 53 | mirror = find_apt_mirror(cloud, cfg) | 53 | mirrors = find_apt_mirror_info(cloud, cfg) |
38 | 54 | if not mirror: | 54 | if not mirrors or "primary" not in mirrors: |
39 | 55 | log.debug(("Skipping module named %s," | 55 | log.debug(("Skipping module named %s," |
40 | 56 | " no package 'mirror' located"), name) | 56 | " no package 'mirror' located"), name) |
41 | 57 | return | 57 | return |
42 | 58 | 58 | ||
44 | 59 | log.debug("Selected mirror at: %s" % mirror) | 59 | # backwards compatibility |
45 | 60 | mirror = mirrors["primary"] | ||
46 | 61 | mirrors["mirror"] = mirror | ||
47 | 62 | |||
48 | 63 | log.debug("mirror info: %s" % mirrors) | ||
49 | 60 | 64 | ||
50 | 61 | if not util.get_cfg_option_bool(cfg, | 65 | if not util.get_cfg_option_bool(cfg, |
51 | 62 | 'apt_preserve_sources_list', False): | 66 | 'apt_preserve_sources_list', False): |
56 | 63 | generate_sources_list(release, mirror, cloud, log) | 67 | generate_sources_list(release, mirrors, cloud, log) |
57 | 64 | old_mir = util.get_cfg_option_str(cfg, 'apt_old_mirror', | 68 | old_mirrors = cfg.get('apt_old_mirrors', |
58 | 65 | "archive.ubuntu.com/ubuntu") | 69 | {"primary": "archive.ubuntu.com/ubuntu", |
59 | 66 | rename_apt_lists(old_mir, mirror) | 70 | "security": "security.ubuntu.com/ubuntu"}) |
60 | 71 | rename_apt_lists(old_mirrors, mirrors) | ||
61 | 67 | 72 | ||
62 | 68 | # Set up any apt proxy | 73 | # Set up any apt proxy |
63 | 69 | proxy = cfg.get("apt_proxy", None) | 74 | proxy = cfg.get("apt_proxy", None) |
64 | @@ -81,8 +86,10 @@ | |||
65 | 81 | 86 | ||
66 | 82 | # Process 'apt_sources' | 87 | # Process 'apt_sources' |
67 | 83 | if 'apt_sources' in cfg: | 88 | if 'apt_sources' in cfg: |
70 | 84 | errors = add_sources(cloud, cfg['apt_sources'], | 89 | params = mirrors |
71 | 85 | {'MIRROR': mirror, 'RELEASE': release}) | 90 | params['RELEASE'] = release |
72 | 91 | params['MIRROR'] = mirror | ||
73 | 92 | errors = add_sources(cloud, cfg['apt_sources'], params) | ||
74 | 86 | for e in errors: | 93 | for e in errors: |
75 | 87 | log.warn("Source Error: %s", ':'.join(e)) | 94 | log.warn("Source Error: %s", ':'.join(e)) |
76 | 88 | 95 | ||
77 | @@ -146,15 +153,18 @@ | |||
78 | 146 | return string | 153 | return string |
79 | 147 | 154 | ||
80 | 148 | 155 | ||
90 | 149 | def rename_apt_lists(omirror, new_mirror, lists_d="/var/lib/apt/lists"): | 156 | def rename_apt_lists(old_mirrors, new_mirrors, lists_d="/var/lib/apt/lists"): |
91 | 150 | oprefix = os.path.join(lists_d, mirror2lists_fileprefix(omirror)) | 157 | for (name, omirror) in old_mirrors.iteritems(): |
92 | 151 | nprefix = os.path.join(lists_d, mirror2lists_fileprefix(new_mirror)) | 158 | nmirror = new_mirrors.get(name) |
93 | 152 | if oprefix == nprefix: | 159 | if not nmirror: |
94 | 153 | return | 160 | continue |
95 | 154 | olen = len(oprefix) | 161 | oprefix = os.path.join(lists_d, mirror2lists_fileprefix(omirror)) |
96 | 155 | for filename in glob.glob("%s_*" % oprefix): | 162 | nprefix = os.path.join(lists_d, mirror2lists_fileprefix(nmirror)) |
97 | 156 | # TODO(harlowja) use the cloud.paths.join... | 163 | if oprefix == nprefix: |
98 | 157 | util.rename(filename, "%s%s" % (nprefix, filename[olen:])) | 164 | continue |
99 | 165 | olen = len(oprefix) | ||
100 | 166 | for filename in glob.glob("%s_*" % oprefix): | ||
101 | 167 | util.rename(filename, "%s%s" % (nprefix, filename[olen:])) | ||
102 | 158 | 168 | ||
103 | 159 | 169 | ||
104 | 160 | def get_release(): | 170 | def get_release(): |
105 | @@ -162,14 +172,17 @@ | |||
106 | 162 | return stdout.strip() | 172 | return stdout.strip() |
107 | 163 | 173 | ||
108 | 164 | 174 | ||
110 | 165 | def generate_sources_list(codename, mirror, cloud, log): | 175 | def generate_sources_list(codename, mirrors, cloud, log): |
111 | 166 | template_fn = cloud.get_template_filename('sources.list') | 176 | template_fn = cloud.get_template_filename('sources.list') |
117 | 167 | if template_fn: | 177 | if not template_fn: |
113 | 168 | params = {'mirror': mirror, 'codename': codename} | ||
114 | 169 | out_fn = cloud.paths.join(False, '/etc/apt/sources.list') | ||
115 | 170 | templater.render_to_file(template_fn, out_fn, params) | ||
116 | 171 | else: | ||
118 | 172 | log.warn("No template found, not rendering /etc/apt/sources.list") | 178 | log.warn("No template found, not rendering /etc/apt/sources.list") |
119 | 179 | return | ||
120 | 180 | |||
121 | 181 | params = {'codename': codename} | ||
122 | 182 | for k in mirrors: | ||
123 | 183 | params[k] = mirrors[k] | ||
124 | 184 | out_fn = cloud.paths.join(False, '/etc/apt/sources.list') | ||
125 | 185 | templater.render_to_file(template_fn, out_fn, params) | ||
126 | 173 | 186 | ||
127 | 174 | 187 | ||
128 | 175 | def add_sources(cloud, srclist, template_params=None): | 188 | def add_sources(cloud, srclist, template_params=None): |
129 | @@ -231,43 +244,47 @@ | |||
130 | 231 | return errorlist | 244 | return errorlist |
131 | 232 | 245 | ||
132 | 233 | 246 | ||
134 | 234 | def find_apt_mirror(cloud, cfg): | 247 | def find_apt_mirror_info(cloud, cfg): |
135 | 235 | """find an apt_mirror given the cloud and cfg provided.""" | 248 | """find an apt_mirror given the cloud and cfg provided.""" |
136 | 236 | 249 | ||
137 | 237 | mirror = None | 250 | mirror = None |
138 | 238 | 251 | ||
147 | 239 | cfg_mirror = cfg.get("apt_mirror", None) | 252 | # this is less preferred way of specifying mirror preferred would be to |
148 | 240 | if cfg_mirror: | 253 | # use the distro's search or package_mirror. |
149 | 241 | mirror = cfg["apt_mirror"] | 254 | mirror = cfg.get("apt_mirror", None) |
150 | 242 | elif "apt_mirror_search" in cfg: | 255 | |
151 | 243 | mirror = util.search_for_mirror(cfg['apt_mirror_search']) | 256 | search = cfg.get("apt_mirror_search", None) |
152 | 244 | else: | 257 | if not mirror and search: |
153 | 245 | mirror = cloud.get_local_mirror() | 258 | mirror = util.search_for_mirror(search) |
154 | 246 | 259 | ||
155 | 260 | if (not mirror and | ||
156 | 261 | util.get_cfg_option_bool(cfg, "apt_mirror_search_dns", False)): | ||
157 | 247 | mydom = "" | 262 | mydom = "" |
158 | 248 | |||
159 | 249 | doms = [] | 263 | doms = [] |
160 | 250 | 264 | ||
184 | 251 | if not mirror: | 265 | # if we have a fqdn, then search its domain portion first |
185 | 252 | # if we have a fqdn, then search its domain portion first | 266 | (_hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) |
186 | 253 | (_hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) | 267 | mydom = ".".join(fqdn.split(".")[1:]) |
187 | 254 | mydom = ".".join(fqdn.split(".")[1:]) | 268 | if mydom: |
188 | 255 | if mydom: | 269 | doms.append(".%s" % mydom) |
189 | 256 | doms.append(".%s" % mydom) | 270 | |
190 | 257 | 271 | doms.extend((".localdomain", "",)) | |
191 | 258 | if (not mirror and | 272 | |
192 | 259 | util.get_cfg_option_bool(cfg, "apt_mirror_search_dns", False)): | 273 | mirror_list = [] |
193 | 260 | doms.extend((".localdomain", "",)) | 274 | distro = cloud.distro.name |
194 | 261 | 275 | mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro) | |
195 | 262 | mirror_list = [] | 276 | for post in doms: |
196 | 263 | distro = cloud.distro.name | 277 | mirror_list.append(mirrorfmt % (post)) |
197 | 264 | mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro) | 278 | |
198 | 265 | for post in doms: | 279 | mirror = util.search_for_mirror(mirror_list) |
199 | 266 | mirror_list.append(mirrorfmt % (post)) | 280 | |
200 | 267 | 281 | mirror_info = cloud.datasource.get_package_mirror_info() | |
201 | 268 | mirror = util.search_for_mirror(mirror_list) | 282 | |
202 | 269 | 283 | # this is a bit strange. | |
203 | 270 | if not mirror: | 284 | # if mirror is set, then one of the legacy options above set it |
204 | 271 | mirror = cloud.distro.get_package_mirror() | 285 | # but they do not cover security. so we need to get that from |
205 | 272 | 286 | # get_package_mirror_info | |
206 | 273 | return mirror | 287 | if mirror: |
207 | 288 | mirror_info.update({'primary': mirror}) | ||
208 | 289 | |||
209 | 290 | return mirror_info | ||
210 | 274 | 291 | ||
211 | === modified file 'cloudinit/distros/__init__.py' | |||
212 | --- cloudinit/distros/__init__.py 2012-08-22 18:12:32 +0000 | |||
213 | +++ cloudinit/distros/__init__.py 2012-08-22 19:21:17 +0000 | |||
214 | @@ -23,6 +23,8 @@ | |||
215 | 23 | from StringIO import StringIO | 23 | from StringIO import StringIO |
216 | 24 | 24 | ||
217 | 25 | import abc | 25 | import abc |
218 | 26 | import os | ||
219 | 27 | import re | ||
220 | 26 | 28 | ||
221 | 27 | from cloudinit import importer | 29 | from cloudinit import importer |
222 | 28 | from cloudinit import log as logging | 30 | from cloudinit import log as logging |
223 | @@ -75,8 +77,26 @@ | |||
224 | 75 | def update_package_sources(self): | 77 | def update_package_sources(self): |
225 | 76 | raise NotImplementedError() | 78 | raise NotImplementedError() |
226 | 77 | 79 | ||
229 | 78 | def get_package_mirror(self): | 80 | def get_primary_arch(self): |
230 | 79 | return self.get_option('package_mirror') | 81 | arch = os.uname[4] |
231 | 82 | if arch in ("i386", "i486", "i586", "i686"): | ||
232 | 83 | return "i386" | ||
233 | 84 | return arch | ||
234 | 85 | |||
235 | 86 | def _get_arch_package_mirror_info(self, arch=None): | ||
236 | 87 | mirror_info = self.get_option("package_mirrors", None) | ||
237 | 88 | if arch == None: | ||
238 | 89 | arch = self.get_primary_arch() | ||
239 | 90 | return _get_arch_package_mirror_info(mirror_info, arch) | ||
240 | 91 | |||
241 | 92 | def get_package_mirror_info(self, arch=None, | ||
242 | 93 | availability_zone=None): | ||
243 | 94 | # this resolves the package_mirrors config option | ||
244 | 95 | # down to a single dict of {mirror_name: mirror_url} | ||
245 | 96 | arch_info = self._get_arch_package_mirror_info(arch) | ||
246 | 97 | |||
247 | 98 | return _get_package_mirror_info(availability_zone=availability_zone, | ||
248 | 99 | mirror_info=arch_info) | ||
249 | 80 | 100 | ||
250 | 81 | def apply_network(self, settings, bring_up=True): | 101 | def apply_network(self, settings, bring_up=True): |
251 | 82 | # Write it out | 102 | # Write it out |
252 | @@ -151,6 +171,55 @@ | |||
253 | 151 | return False | 171 | return False |
254 | 152 | 172 | ||
255 | 153 | 173 | ||
256 | 174 | def _get_package_mirror_info(mirror_info, availability_zone=None, | ||
257 | 175 | mirror_filter=util.search_for_mirror): | ||
258 | 176 | # given a arch specific 'mirror_info' entry (from package_mirrors) | ||
259 | 177 | # search through the 'search' entries, and fallback appropriately | ||
260 | 178 | # return a dict with only {name: mirror} entries. | ||
261 | 179 | |||
262 | 180 | ec2_az_re = ("^[a-z][a-z]-(%s)-[1-9][0-9]*[a-z]$" % | ||
263 | 181 | "north|northeast|east|southeast|south|southwest|west|northwest") | ||
264 | 182 | |||
265 | 183 | subst = {} | ||
266 | 184 | if availability_zone: | ||
267 | 185 | subst['availability_zone'] = availability_zone | ||
268 | 186 | |||
269 | 187 | if availability_zone and re.match(ec2_az_re, availability_zone): | ||
270 | 188 | subst['ec2_region'] = "%s" % availability_zone[0:-1] | ||
271 | 189 | |||
272 | 190 | results = {} | ||
273 | 191 | for (name, mirror) in mirror_info.get('failsafe', {}).iteritems(): | ||
274 | 192 | results[name] = mirror | ||
275 | 193 | |||
276 | 194 | for (name, searchlist) in mirror_info.get('search', {}).iteritems(): | ||
277 | 195 | mirrors = [] | ||
278 | 196 | for tmpl in searchlist: | ||
279 | 197 | try: | ||
280 | 198 | mirrors.append(tmpl % subst) | ||
281 | 199 | except KeyError: | ||
282 | 200 | pass | ||
283 | 201 | |||
284 | 202 | found = mirror_filter(mirrors) | ||
285 | 203 | if found: | ||
286 | 204 | results[name] = found | ||
287 | 205 | |||
288 | 206 | LOG.debug("filtered distro mirror info: %s" % results) | ||
289 | 207 | |||
290 | 208 | return results | ||
291 | 209 | |||
292 | 210 | |||
293 | 211 | def _get_arch_package_mirror_info(package_mirrors, arch): | ||
294 | 212 | # pull out the specific arch from a 'package_mirrors' config option | ||
295 | 213 | default = None | ||
296 | 214 | for item in package_mirrors: | ||
297 | 215 | arches = item.get("arches") | ||
298 | 216 | if arch in arches: | ||
299 | 217 | return item | ||
300 | 218 | if "default" in arches: | ||
301 | 219 | default = item | ||
302 | 220 | return default | ||
303 | 221 | |||
304 | 222 | |||
305 | 154 | def fetch(name): | 223 | def fetch(name): |
306 | 155 | locs = importer.find_module(name, | 224 | locs = importer.find_module(name, |
307 | 156 | ['', __name__], | 225 | ['', __name__], |
308 | 157 | 226 | ||
309 | === modified file 'cloudinit/distros/debian.py' | |||
310 | --- cloudinit/distros/debian.py 2012-06-30 00:06:32 +0000 | |||
311 | +++ cloudinit/distros/debian.py 2012-08-22 19:21:17 +0000 | |||
312 | @@ -147,3 +147,7 @@ | |||
313 | 147 | def update_package_sources(self): | 147 | def update_package_sources(self): |
314 | 148 | self._runner.run("update-sources", self.package_command, | 148 | self._runner.run("update-sources", self.package_command, |
315 | 149 | ["update"], freq=PER_INSTANCE) | 149 | ["update"], freq=PER_INSTANCE) |
316 | 150 | |||
317 | 151 | def get_primary_arch(self): | ||
318 | 152 | (arch, _err) = util.subp(['dpkg', '--print-architecture']) | ||
319 | 153 | return str(arch).strip() | ||
320 | 150 | 154 | ||
321 | === modified file 'cloudinit/sources/DataSourceCloudStack.py' | |||
322 | --- cloudinit/sources/DataSourceCloudStack.py 2012-08-22 18:12:32 +0000 | |||
323 | +++ cloudinit/sources/DataSourceCloudStack.py 2012-08-22 19:21:17 +0000 | |||
324 | @@ -131,7 +131,8 @@ | |||
325 | 131 | def get_instance_id(self): | 131 | def get_instance_id(self): |
326 | 132 | return self.metadata['instance-id'] | 132 | return self.metadata['instance-id'] |
327 | 133 | 133 | ||
329 | 134 | def get_availability_zone(self): | 134 | @property |
330 | 135 | def availability_zone(self): | ||
331 | 135 | return self.metadata['availability-zone'] | 136 | return self.metadata['availability-zone'] |
332 | 136 | 137 | ||
333 | 137 | 138 | ||
334 | 138 | 139 | ||
335 | === modified file 'cloudinit/sources/DataSourceEc2.py' | |||
336 | --- cloudinit/sources/DataSourceEc2.py 2012-07-16 20:16:27 +0000 | |||
337 | +++ cloudinit/sources/DataSourceEc2.py 2012-08-22 19:21:17 +0000 | |||
338 | @@ -83,40 +83,6 @@ | |||
339 | 83 | def get_availability_zone(self): | 83 | def get_availability_zone(self): |
340 | 84 | return self.metadata['placement']['availability-zone'] | 84 | return self.metadata['placement']['availability-zone'] |
341 | 85 | 85 | ||
342 | 86 | def get_local_mirror(self): | ||
343 | 87 | return self.get_mirror_from_availability_zone() | ||
344 | 88 | |||
345 | 89 | def get_mirror_from_availability_zone(self, availability_zone=None): | ||
346 | 90 | # Return type None indicates there is no cloud specific mirror | ||
347 | 91 | # Availability is like 'us-west-1b' or 'eu-west-1a' | ||
348 | 92 | if availability_zone is None: | ||
349 | 93 | availability_zone = self.get_availability_zone() | ||
350 | 94 | |||
351 | 95 | if self.is_vpc(): | ||
352 | 96 | return None | ||
353 | 97 | |||
354 | 98 | if not availability_zone: | ||
355 | 99 | return None | ||
356 | 100 | |||
357 | 101 | mirror_tpl = self.distro.get_option('package_mirror_ec2_template', | ||
358 | 102 | None) | ||
359 | 103 | |||
360 | 104 | if mirror_tpl is None: | ||
361 | 105 | return None | ||
362 | 106 | |||
363 | 107 | # in EC2, the 'region' is 'us-east-1' if 'zone' is 'us-east-1a' | ||
364 | 108 | tpl_params = { | ||
365 | 109 | 'zone': availability_zone.strip(), | ||
366 | 110 | 'region': availability_zone[:-1] | ||
367 | 111 | } | ||
368 | 112 | mirror_url = mirror_tpl % (tpl_params) | ||
369 | 113 | |||
370 | 114 | found = util.search_for_mirror([mirror_url]) | ||
371 | 115 | if found is not None: | ||
372 | 116 | return mirror_url | ||
373 | 117 | |||
374 | 118 | return None | ||
375 | 119 | |||
376 | 120 | def _get_url_settings(self): | 86 | def _get_url_settings(self): |
377 | 121 | mcfg = self.ds_cfg | 87 | mcfg = self.ds_cfg |
378 | 122 | if not mcfg: | 88 | if not mcfg: |
379 | @@ -255,6 +221,12 @@ | |||
380 | 255 | return True | 221 | return True |
381 | 256 | return False | 222 | return False |
382 | 257 | 223 | ||
383 | 224 | @property | ||
384 | 225 | def availability_zone(self): | ||
385 | 226 | try: | ||
386 | 227 | return self.metadata['placement']['availability-zone'] | ||
387 | 228 | except KeyError: | ||
388 | 229 | return None | ||
389 | 258 | 230 | ||
390 | 259 | # Used to match classes to dependencies | 231 | # Used to match classes to dependencies |
391 | 260 | datasources = [ | 232 | datasources = [ |
392 | 261 | 233 | ||
393 | === modified file 'cloudinit/sources/__init__.py' | |||
394 | --- cloudinit/sources/__init__.py 2012-08-20 05:28:14 +0000 | |||
395 | +++ cloudinit/sources/__init__.py 2012-08-22 19:21:17 +0000 | |||
396 | @@ -117,9 +117,9 @@ | |||
397 | 117 | def get_locale(self): | 117 | def get_locale(self): |
398 | 118 | return 'en_US.UTF-8' | 118 | return 'en_US.UTF-8' |
399 | 119 | 119 | ||
403 | 120 | def get_local_mirror(self): | 120 | @property |
404 | 121 | # ?? | 121 | def availability_zone(self): |
405 | 122 | return None | 122 | return self.metadata.get('availability-zone') |
406 | 123 | 123 | ||
407 | 124 | def get_instance_id(self): | 124 | def get_instance_id(self): |
408 | 125 | if not self.metadata or 'instance-id' not in self.metadata: | 125 | if not self.metadata or 'instance-id' not in self.metadata: |
409 | @@ -166,6 +166,10 @@ | |||
410 | 166 | else: | 166 | else: |
411 | 167 | return hostname | 167 | return hostname |
412 | 168 | 168 | ||
413 | 169 | def get_package_mirror_info(self): | ||
414 | 170 | return self.distro.get_package_mirror_info( | ||
415 | 171 | availability_zone=self.availability_zone) | ||
416 | 172 | |||
417 | 169 | 173 | ||
418 | 170 | def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list): | 174 | def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list): |
419 | 171 | ds_list = list_sources(cfg_list, ds_deps, pkg_list) | 175 | ds_list = list_sources(cfg_list, ds_deps, pkg_list) |
420 | 172 | 176 | ||
421 | === modified file 'config/cloud.cfg' | |||
422 | --- config/cloud.cfg 2012-08-20 21:11:46 +0000 | |||
423 | +++ config/cloud.cfg 2012-08-22 19:21:17 +0000 | |||
424 | @@ -74,6 +74,18 @@ | |||
425 | 74 | cloud_dir: /var/lib/cloud/ | 74 | cloud_dir: /var/lib/cloud/ |
426 | 75 | templates_dir: /etc/cloud/templates/ | 75 | templates_dir: /etc/cloud/templates/ |
427 | 76 | upstart_dir: /etc/init/ | 76 | upstart_dir: /etc/init/ |
430 | 77 | package_mirror: http://archive.ubuntu.com/ubuntu | 77 | package_mirrors: |
431 | 78 | package_mirror_ec2_template: http://%(region)s.ec2.archive.ubuntu.com/ubuntu/ | 78 | - arches: [i386, amd64] |
432 | 79 | failsafe: | ||
433 | 80 | primary: http://archive.ubuntu.com/ubuntu | ||
434 | 81 | security: http://security.ubuntu.com/ubuntu | ||
435 | 82 | search: | ||
436 | 83 | primary: | ||
437 | 84 | - http://%(ec2_region)s.ec2.archive.ubuntu.com/ubuntu/ | ||
438 | 85 | - http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/ | ||
439 | 86 | security: [] | ||
440 | 87 | - arches: [armhf, armel, default] | ||
441 | 88 | failsafe: | ||
442 | 89 | primary: http://ports.ubuntu.com/ubuntu | ||
443 | 90 | security: http://ports.ubuntu.com/ubuntu | ||
444 | 79 | ssh_svcname: ssh | 91 | ssh_svcname: ssh |
445 | 80 | 92 | ||
446 | === modified file 'templates/sources.list.tmpl' | |||
447 | --- templates/sources.list.tmpl 2012-07-09 20:41:45 +0000 | |||
448 | +++ templates/sources.list.tmpl 2012-08-22 19:21:17 +0000 | |||
449 | @@ -52,9 +52,9 @@ | |||
450 | 52 | # deb http://archive.canonical.com/ubuntu $codename partner | 52 | # deb http://archive.canonical.com/ubuntu $codename partner |
451 | 53 | # deb-src http://archive.canonical.com/ubuntu $codename partner | 53 | # deb-src http://archive.canonical.com/ubuntu $codename partner |
452 | 54 | 54 | ||
459 | 55 | deb http://security.ubuntu.com/ubuntu $codename-security main | 55 | deb $security $codename-security main |
460 | 56 | deb-src http://security.ubuntu.com/ubuntu $codename-security main | 56 | deb-src $security $codename-security main |
461 | 57 | deb http://security.ubuntu.com/ubuntu $codename-security universe | 57 | deb $security $codename-security universe |
462 | 58 | deb-src http://security.ubuntu.com/ubuntu $codename-security universe | 58 | deb-src $security $codename-security universe |
463 | 59 | # deb http://security.ubuntu.com/ubuntu $codename-security multiverse | 59 | # deb $security $codename-security multiverse |
464 | 60 | # deb-src http://security.ubuntu.com/ubuntu $codename-security multiverse | 60 | # deb-src $security $codename-security multiverse |
465 | 61 | 61 | ||
466 | === added directory 'tests/unittests/test_distros' | |||
467 | === added file 'tests/unittests/test_distros/test_generic.py' | |||
468 | --- tests/unittests/test_distros/test_generic.py 1970-01-01 00:00:00 +0000 | |||
469 | +++ tests/unittests/test_distros/test_generic.py 2012-08-22 19:21:17 +0000 | |||
470 | @@ -0,0 +1,121 @@ | |||
471 | 1 | from mocker import MockerTestCase | ||
472 | 2 | |||
473 | 3 | from cloudinit import distros | ||
474 | 4 | |||
475 | 5 | unknown_arch_info = { | ||
476 | 6 | 'arches': ['default'], | ||
477 | 7 | 'failsafe': {'primary': 'http://fs-primary-default', | ||
478 | 8 | 'security': 'http://fs-security-default'} | ||
479 | 9 | } | ||
480 | 10 | |||
481 | 11 | package_mirrors = [ | ||
482 | 12 | {'arches': ['i386', 'amd64'], | ||
483 | 13 | 'failsafe': {'primary': 'http://fs-primary-intel', | ||
484 | 14 | 'security': 'http://fs-security-intel'}, | ||
485 | 15 | 'search': { | ||
486 | 16 | 'primary': ['http://%(ec2_region)s.ec2/', | ||
487 | 17 | 'http://%(availability_zone)s.clouds/'], | ||
488 | 18 | 'security': ['http://security-mirror1-intel', | ||
489 | 19 | 'http://security-mirror2-intel']}}, | ||
490 | 20 | {'arches': ['armhf', 'armel'], | ||
491 | 21 | 'failsafe': {'primary': 'http://fs-primary-arm', | ||
492 | 22 | 'security': 'http://fs-security-arm'}}, | ||
493 | 23 | unknown_arch_info | ||
494 | 24 | ] | ||
495 | 25 | |||
496 | 26 | gpmi = distros._get_package_mirror_info # pylint: disable=W0212 | ||
497 | 27 | gapmi = distros._get_arch_package_mirror_info # pylint: disable=W0212 | ||
498 | 28 | |||
499 | 29 | |||
500 | 30 | class TestGenericDistro(MockerTestCase): | ||
501 | 31 | |||
502 | 32 | def return_first(self, mlist): | ||
503 | 33 | if not mlist: | ||
504 | 34 | return None | ||
505 | 35 | return mlist[0] | ||
506 | 36 | |||
507 | 37 | def return_second(self, mlist): | ||
508 | 38 | if not mlist: | ||
509 | 39 | return None | ||
510 | 40 | return mlist[1] | ||
511 | 41 | |||
512 | 42 | def return_none(self, _mlist): | ||
513 | 43 | return None | ||
514 | 44 | |||
515 | 45 | def return_last(self, mlist): | ||
516 | 46 | if not mlist: | ||
517 | 47 | return None | ||
518 | 48 | return(mlist[-1]) | ||
519 | 49 | |||
520 | 50 | def setUp(self): | ||
521 | 51 | super(TestGenericDistro, self).setUp() | ||
522 | 52 | # Make a temp directoy for tests to use. | ||
523 | 53 | self.tmp = self.makeDir() | ||
524 | 54 | |||
525 | 55 | def test_arch_package_mirror_info_unknown(self): | ||
526 | 56 | """for an unknown arch, we should get back that with arch 'default'.""" | ||
527 | 57 | arch_mirrors = gapmi(package_mirrors, arch="unknown") | ||
528 | 58 | self.assertEqual(unknown_arch_info, arch_mirrors) | ||
529 | 59 | |||
530 | 60 | def test_arch_package_mirror_info_known(self): | ||
531 | 61 | arch_mirrors = gapmi(package_mirrors, arch="amd64") | ||
532 | 62 | self.assertEqual(package_mirrors[0], arch_mirrors) | ||
533 | 63 | |||
534 | 64 | def test_get_package_mirror_info_az_ec2(self): | ||
535 | 65 | arch_mirrors = gapmi(package_mirrors, arch="amd64") | ||
536 | 66 | |||
537 | 67 | results = gpmi(arch_mirrors, availability_zone="us-east-1a", | ||
538 | 68 | mirror_filter=self.return_first) | ||
539 | 69 | self.assertEqual(results, | ||
540 | 70 | {'primary': 'http://us-east-1.ec2/', | ||
541 | 71 | 'security': 'http://security-mirror1-intel'}) | ||
542 | 72 | |||
543 | 73 | results = gpmi(arch_mirrors, availability_zone="us-east-1a", | ||
544 | 74 | mirror_filter=self.return_second) | ||
545 | 75 | self.assertEqual(results, | ||
546 | 76 | {'primary': 'http://us-east-1a.clouds/', | ||
547 | 77 | 'security': 'http://security-mirror2-intel'}) | ||
548 | 78 | |||
549 | 79 | results = gpmi(arch_mirrors, availability_zone="us-east-1a", | ||
550 | 80 | mirror_filter=self.return_none) | ||
551 | 81 | self.assertEqual(results, package_mirrors[0]['failsafe']) | ||
552 | 82 | |||
553 | 83 | def test_get_package_mirror_info_az_non_ec2(self): | ||
554 | 84 | arch_mirrors = gapmi(package_mirrors, arch="amd64") | ||
555 | 85 | |||
556 | 86 | results = gpmi(arch_mirrors, availability_zone="nova.cloudvendor", | ||
557 | 87 | mirror_filter=self.return_first) | ||
558 | 88 | self.assertEqual(results, | ||
559 | 89 | {'primary': 'http://nova.cloudvendor.clouds/', | ||
560 | 90 | 'security': 'http://security-mirror1-intel'}) | ||
561 | 91 | |||
562 | 92 | results = gpmi(arch_mirrors, availability_zone="nova.cloudvendor", | ||
563 | 93 | mirror_filter=self.return_last) | ||
564 | 94 | self.assertEqual(results, | ||
565 | 95 | {'primary': 'http://nova.cloudvendor.clouds/', | ||
566 | 96 | 'security': 'http://security-mirror2-intel'}) | ||
567 | 97 | |||
568 | 98 | def test_get_package_mirror_info_none(self): | ||
569 | 99 | arch_mirrors = gapmi(package_mirrors, arch="amd64") | ||
570 | 100 | |||
571 | 101 | # because both search entries here replacement based on | ||
572 | 102 | # availability-zone, the filter will be called with an empty list and | ||
573 | 103 | # failsafe should be taken. | ||
574 | 104 | results = gpmi(arch_mirrors, availability_zone=None, | ||
575 | 105 | mirror_filter=self.return_first) | ||
576 | 106 | self.assertEqual(results, | ||
577 | 107 | {'primary': 'http://fs-primary-intel', | ||
578 | 108 | 'security': 'http://security-mirror1-intel'}) | ||
579 | 109 | |||
580 | 110 | results = gpmi(arch_mirrors, availability_zone=None, | ||
581 | 111 | mirror_filter=self.return_last) | ||
582 | 112 | self.assertEqual(results, | ||
583 | 113 | {'primary': 'http://fs-primary-intel', | ||
584 | 114 | 'security': 'http://security-mirror2-intel'}) | ||
585 | 115 | |||
586 | 116 | |||
587 | 117 | #def _get_package_mirror_info(mirror_info, availability_zone=None, | ||
588 | 118 | # mirror_filter=util.search_for_mirror): | ||
589 | 119 | |||
590 | 120 | |||
591 | 121 | # vi: ts=4 expandtab |