Merge lp:~cjwatson/launchpad/explicit-proxy-mirror-prober into lp:launchpad
- explicit-proxy-mirror-prober
- Merge into devel
Proposed by
Colin Watson
Status: | Merged |
---|---|
Merged at revision: | 18712 |
Proposed branch: | lp:~cjwatson/launchpad/explicit-proxy-mirror-prober |
Merge into: | lp:launchpad |
Prerequisite: | lp:~cjwatson/launchpad/no-default-http-proxy |
Diff against target: |
585 lines (+167/-110) 5 files modified
cronscripts/distributionmirror-prober.py (+4/-9) lib/lp/registry/scripts/distributionmirror_prober.py (+24/-29) lib/lp/registry/tests/test_distributionmirror_prober.py (+81/-50) lib/lp/services/tests/test_timeout.py (+50/-20) lib/lp/services/timeout.py (+8/-2) |
To merge this branch: | bzr merge lp:~cjwatson/launchpad/explicit-proxy-mirror-prober |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
William Grant | code | Approve | |
Review via email: mp+348543@code.launchpad.net |
Commit message
Convert the mirror prober to use explicit proxy configuration.
Description of the change
To post a comment you must log in.
Revision history for this message
William Grant (wgrant) : | # |
review:
Approve
(code)
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'cronscripts/distributionmirror-prober.py' | |||
2 | --- cronscripts/distributionmirror-prober.py 2013-01-07 02:40:55 +0000 | |||
3 | +++ cronscripts/distributionmirror-prober.py 2018-07-02 11:30:55 +0000 | |||
4 | @@ -1,14 +1,12 @@ | |||
5 | 1 | #!/usr/bin/python -S | 1 | #!/usr/bin/python -S |
6 | 2 | # | 2 | # |
8 | 3 | # Copyright 2009-2011 Canonical Ltd. This software is licensed under the | 3 | # Copyright 2009-2018 Canonical Ltd. This software is licensed under the |
9 | 4 | # GNU Affero General Public License version 3 (see the file LICENSE). | 4 | # GNU Affero General Public License version 3 (see the file LICENSE). |
10 | 5 | 5 | ||
11 | 6 | """Script to probe distribution mirrors and check how up-to-date they are.""" | 6 | """Script to probe distribution mirrors and check how up-to-date they are.""" |
12 | 7 | 7 | ||
13 | 8 | import _pythonpath | 8 | import _pythonpath |
14 | 9 | 9 | ||
15 | 10 | import os | ||
16 | 11 | |||
17 | 12 | from lp.registry.interfaces.distributionmirror import MirrorContent | 10 | from lp.registry.interfaces.distributionmirror import MirrorContent |
18 | 13 | from lp.registry.scripts.distributionmirror_prober import DistroMirrorProber | 11 | from lp.registry.scripts.distributionmirror_prober import DistroMirrorProber |
19 | 14 | from lp.services.config import config | 12 | from lp.services.config import config |
20 | @@ -16,6 +14,7 @@ | |||
21 | 16 | LaunchpadCronScript, | 14 | LaunchpadCronScript, |
22 | 17 | LaunchpadScriptFailure, | 15 | LaunchpadScriptFailure, |
23 | 18 | ) | 16 | ) |
24 | 17 | from lp.services.timeout import set_default_timeout_function | ||
25 | 19 | 18 | ||
26 | 20 | 19 | ||
27 | 21 | class DistroMirrorProberScript(LaunchpadCronScript): | 20 | class DistroMirrorProberScript(LaunchpadCronScript): |
28 | @@ -49,12 +48,8 @@ | |||
29 | 49 | 'Wrong value for argument --content-type: %s' | 48 | 'Wrong value for argument --content-type: %s' |
30 | 50 | % self.options.content_type) | 49 | % self.options.content_type) |
31 | 51 | 50 | ||
38 | 52 | if config.distributionmirrorprober.use_proxy: | 51 | set_default_timeout_function( |
39 | 53 | os.environ['http_proxy'] = config.launchpad.http_proxy | 52 | lambda: config.distributionmirrorprober.timeout) |
34 | 54 | self.logger.debug("Using %s as proxy." % os.environ['http_proxy']) | ||
35 | 55 | else: | ||
36 | 56 | self.logger.debug("Not using any proxy.") | ||
37 | 57 | |||
40 | 58 | DistroMirrorProber(self.txn, self.logger).probe( | 53 | DistroMirrorProber(self.txn, self.logger).probe( |
41 | 59 | content_type, self.options.no_remote_hosts, self.options.force, | 54 | content_type, self.options.no_remote_hosts, self.options.force, |
42 | 60 | self.options.max_mirrors, not self.options.no_owner_notification) | 55 | self.options.max_mirrors, not self.options.no_owner_notification) |
43 | 61 | 56 | ||
44 | === modified file 'lib/lp/registry/scripts/distributionmirror_prober.py' | |||
45 | --- lib/lp/registry/scripts/distributionmirror_prober.py 2017-05-22 09:33:45 +0000 | |||
46 | +++ lib/lp/registry/scripts/distributionmirror_prober.py 2018-07-02 11:30:55 +0000 | |||
47 | @@ -1,4 +1,4 @@ | |||
49 | 1 | # Copyright 2009-2017 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2018 Canonical Ltd. This software is licensed under the |
50 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
51 | 3 | 3 | ||
52 | 4 | __metaclass__ = type | 4 | __metaclass__ = type |
53 | @@ -10,12 +10,12 @@ | |||
54 | 10 | import httplib | 10 | import httplib |
55 | 11 | import itertools | 11 | import itertools |
56 | 12 | import logging | 12 | import logging |
58 | 13 | import os | 13 | import os.path |
59 | 14 | from StringIO import StringIO | 14 | from StringIO import StringIO |
60 | 15 | import urllib | 15 | import urllib |
61 | 16 | import urllib2 | ||
62 | 17 | import urlparse | 16 | import urlparse |
63 | 18 | 17 | ||
64 | 18 | import requests | ||
65 | 19 | from twisted.internet import ( | 19 | from twisted.internet import ( |
66 | 20 | defer, | 20 | defer, |
67 | 21 | protocol, | 21 | protocol, |
68 | @@ -36,6 +36,7 @@ | |||
69 | 36 | from lp.registry.interfaces.distroseries import IDistroSeries | 36 | from lp.registry.interfaces.distroseries import IDistroSeries |
70 | 37 | from lp.services.config import config | 37 | from lp.services.config import config |
71 | 38 | from lp.services.librarian.interfaces import ILibraryFileAliasSet | 38 | from lp.services.librarian.interfaces import ILibraryFileAliasSet |
72 | 39 | from lp.services.timeout import urlfetch | ||
73 | 39 | from lp.services.webapp import canonical_url | 40 | from lp.services.webapp import canonical_url |
74 | 40 | from lp.soyuz.interfaces.distroarchseries import IDistroArchSeries | 41 | from lp.soyuz.interfaces.distroarchseries import IDistroArchSeries |
75 | 41 | 42 | ||
76 | @@ -202,9 +203,9 @@ | |||
77 | 202 | request_path = None | 203 | request_path = None |
78 | 203 | 204 | ||
79 | 204 | # Details of the URL of the host in which we'll connect, which will only | 205 | # Details of the URL of the host in which we'll connect, which will only |
83 | 205 | # be different from request_* in case we have an http_proxy environment | 206 | # be different from request_* in case we have a configured http_proxy -- |
84 | 206 | # variable --in that case the scheme, host and port will be the ones | 207 | # in that case the scheme, host and port will be the ones extracted from |
85 | 207 | # extracted from http_proxy and the path will be self.url | 208 | # http_proxy and the path will be self.url. |
86 | 208 | connect_scheme = None | 209 | connect_scheme = None |
87 | 209 | connect_host = None | 210 | connect_host = None |
88 | 210 | connect_port = None | 211 | connect_port = None |
89 | @@ -279,7 +280,7 @@ | |||
90 | 279 | # XXX Guilherme Salgado 2006-09-19: | 280 | # XXX Guilherme Salgado 2006-09-19: |
91 | 280 | # We don't actually know how to handle FTP responses, but we | 281 | # We don't actually know how to handle FTP responses, but we |
92 | 281 | # expect to be behind a squid HTTP proxy with the patch at | 282 | # expect to be behind a squid HTTP proxy with the patch at |
94 | 282 | # http://www.squid-cache.org/bugs/show_bug.cgi?id=1758 applied. So, if | 283 | # https://bugs.squid-cache.org/show_bug.cgi?id=1758 applied. So, if |
95 | 283 | # you encounter any problems with FTP URLs you'll probably have to nag | 284 | # you encounter any problems with FTP URLs you'll probably have to nag |
96 | 284 | # the sysadmins to fix squid for you. | 285 | # the sysadmins to fix squid for you. |
97 | 285 | if scheme not in ('http', 'ftp'): | 286 | if scheme not in ('http', 'ftp'): |
98 | @@ -296,9 +297,9 @@ | |||
99 | 296 | if self.request_host not in host_timeouts: | 297 | if self.request_host not in host_timeouts: |
100 | 297 | host_timeouts[self.request_host] = 0 | 298 | host_timeouts[self.request_host] = 0 |
101 | 298 | 299 | ||
105 | 299 | # If the http_proxy variable is set, we want to use it as the host | 300 | # If launchpad.http_proxy is set in our configuration, we want to |
106 | 300 | # we're going to connect to. | 301 | # use it as the host we're going to connect to. |
107 | 301 | proxy = os.getenv('http_proxy') | 302 | proxy = config.launchpad.http_proxy |
108 | 302 | if proxy: | 303 | if proxy: |
109 | 303 | scheme, host, port, dummy = _parse(proxy) | 304 | scheme, host, port, dummy = _parse(proxy) |
110 | 304 | path = url | 305 | path = url |
111 | @@ -612,31 +613,25 @@ | |||
112 | 612 | return failure | 613 | return failure |
113 | 613 | 614 | ||
114 | 614 | 615 | ||
115 | 615 | def _build_request_for_cdimage_file_list(url): | ||
116 | 616 | headers = {'Pragma': 'no-cache', 'Cache-control': 'no-cache'} | ||
117 | 617 | return urllib2.Request(url, headers=headers) | ||
118 | 618 | |||
119 | 619 | |||
120 | 620 | def _get_cdimage_file_list(): | 616 | def _get_cdimage_file_list(): |
121 | 621 | url = config.distributionmirrorprober.cdimage_file_list_url | 617 | url = config.distributionmirrorprober.cdimage_file_list_url |
122 | 618 | # In test environments, this may be a file: URL. Adjust it to be in a | ||
123 | 619 | # form that requests can cope with (i.e. using an absolute path). | ||
124 | 620 | parsed_url = urlparse.urlparse(url) | ||
125 | 621 | if parsed_url.scheme == 'file' and not os.path.isabs(parsed_url.path): | ||
126 | 622 | assert parsed_url.path == parsed_url[2] | ||
127 | 623 | parsed_url = list(parsed_url) | ||
128 | 624 | parsed_url[2] = os.path.join(config.root, parsed_url[2]) | ||
129 | 625 | url = urlparse.urlunparse(parsed_url) | ||
130 | 622 | try: | 626 | try: |
133 | 623 | return urllib2.urlopen(_build_request_for_cdimage_file_list(url)) | 627 | return urlfetch( |
134 | 624 | except urllib2.URLError as e: | 628 | url, headers={'Pragma': 'no-cache', 'Cache-control': 'no-cache'}, |
135 | 629 | trust_env=False, use_proxy=True, allow_file=True) | ||
136 | 630 | except requests.RequestException as e: | ||
137 | 625 | raise UnableToFetchCDImageFileList( | 631 | raise UnableToFetchCDImageFileList( |
138 | 626 | 'Unable to fetch %s: %s' % (url, e)) | 632 | 'Unable to fetch %s: %s' % (url, e)) |
139 | 627 | 633 | ||
140 | 628 | 634 | ||
141 | 629 | def restore_http_proxy(http_proxy): | ||
142 | 630 | """Restore the http_proxy environment variable to the given value.""" | ||
143 | 631 | if http_proxy is None: | ||
144 | 632 | try: | ||
145 | 633 | del os.environ['http_proxy'] | ||
146 | 634 | except KeyError: | ||
147 | 635 | pass | ||
148 | 636 | else: | ||
149 | 637 | os.environ['http_proxy'] = http_proxy | ||
150 | 638 | |||
151 | 639 | |||
152 | 640 | def get_expected_cdimage_paths(): | 635 | def get_expected_cdimage_paths(): |
153 | 641 | """Get all paths where we can find CD image files on a cdimage mirror. | 636 | """Get all paths where we can find CD image files on a cdimage mirror. |
154 | 642 | 637 | ||
155 | @@ -648,7 +643,7 @@ | |||
156 | 648 | UnableToFetchCDImageFileList exception will be raised. | 643 | UnableToFetchCDImageFileList exception will be raised. |
157 | 649 | """ | 644 | """ |
158 | 650 | d = {} | 645 | d = {} |
160 | 651 | for line in _get_cdimage_file_list().readlines(): | 646 | for line in _get_cdimage_file_list().iter_lines(): |
161 | 652 | flavour, seriesname, path, size = line.split('\t') | 647 | flavour, seriesname, path, size = line.split('\t') |
162 | 653 | paths = d.setdefault((flavour, seriesname), []) | 648 | paths = d.setdefault((flavour, seriesname), []) |
163 | 654 | paths.append(path.lstrip('/')) | 649 | paths.append(path.lstrip('/')) |
164 | 655 | 650 | ||
165 | === modified file 'lib/lp/registry/tests/test_distributionmirror_prober.py' | |||
166 | --- lib/lp/registry/tests/test_distributionmirror_prober.py 2018-01-02 16:10:26 +0000 | |||
167 | +++ lib/lp/registry/tests/test_distributionmirror_prober.py 2018-07-02 11:30:55 +0000 | |||
168 | @@ -1,4 +1,4 @@ | |||
170 | 1 | # Copyright 2009-2016 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2018 Canonical Ltd. This software is licensed under the |
171 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
172 | 3 | 3 | ||
173 | 4 | """distributionmirror-prober tests.""" | 4 | """distributionmirror-prober tests.""" |
174 | @@ -13,13 +13,23 @@ | |||
175 | 13 | from StringIO import StringIO | 13 | from StringIO import StringIO |
176 | 14 | 14 | ||
177 | 15 | from lazr.uri import URI | 15 | from lazr.uri import URI |
178 | 16 | import responses | ||
179 | 16 | from sqlobject import SQLObjectNotFound | 17 | from sqlobject import SQLObjectNotFound |
180 | 18 | from testtools.matchers import ( | ||
181 | 19 | ContainsDict, | ||
182 | 20 | Equals, | ||
183 | 21 | MatchesStructure, | ||
184 | 22 | ) | ||
185 | 23 | from testtools.twistedsupport import ( | ||
186 | 24 | assert_fails_with, | ||
187 | 25 | AsynchronousDeferredRunTest, | ||
188 | 26 | AsynchronousDeferredRunTestForBrokenTwisted, | ||
189 | 27 | ) | ||
190 | 17 | from twisted.internet import ( | 28 | from twisted.internet import ( |
191 | 18 | defer, | 29 | defer, |
192 | 19 | reactor, | 30 | reactor, |
193 | 20 | ) | 31 | ) |
194 | 21 | from twisted.python.failure import Failure | 32 | from twisted.python.failure import Failure |
195 | 22 | from twisted.trial.unittest import TestCase as TrialTestCase | ||
196 | 23 | from twisted.web import server | 33 | from twisted.web import server |
197 | 24 | from zope.component import getUtility | 34 | from zope.component import getUtility |
198 | 25 | from zope.security.proxy import removeSecurityProxy | 35 | from zope.security.proxy import removeSecurityProxy |
199 | @@ -29,7 +39,7 @@ | |||
200 | 29 | from lp.registry.model.distributionmirror import DistributionMirror | 39 | from lp.registry.model.distributionmirror import DistributionMirror |
201 | 30 | from lp.registry.scripts import distributionmirror_prober | 40 | from lp.registry.scripts import distributionmirror_prober |
202 | 31 | from lp.registry.scripts.distributionmirror_prober import ( | 41 | from lp.registry.scripts.distributionmirror_prober import ( |
204 | 32 | _build_request_for_cdimage_file_list, | 42 | _get_cdimage_file_list, |
205 | 33 | ArchiveMirrorProberCallbacks, | 43 | ArchiveMirrorProberCallbacks, |
206 | 34 | BadResponseCode, | 44 | BadResponseCode, |
207 | 35 | ConnectionSkipped, | 45 | ConnectionSkipped, |
208 | @@ -50,7 +60,6 @@ | |||
209 | 50 | RedirectAwareProberProtocol, | 60 | RedirectAwareProberProtocol, |
210 | 51 | RedirectToDifferentFile, | 61 | RedirectToDifferentFile, |
211 | 52 | RequestManager, | 62 | RequestManager, |
212 | 53 | restore_http_proxy, | ||
213 | 54 | should_skip_host, | 63 | should_skip_host, |
214 | 55 | UnknownURLSchemeAfterRedirect, | 64 | UnknownURLSchemeAfterRedirect, |
215 | 56 | ) | 65 | ) |
216 | @@ -59,6 +68,7 @@ | |||
217 | 59 | ) | 68 | ) |
218 | 60 | from lp.services.config import config | 69 | from lp.services.config import config |
219 | 61 | from lp.services.daemons.tachandler import TacTestSetup | 70 | from lp.services.daemons.tachandler import TacTestSetup |
220 | 71 | from lp.services.timeout import default_timeout | ||
221 | 62 | from lp.testing import ( | 72 | from lp.testing import ( |
222 | 63 | clean_up_reactor, | 73 | clean_up_reactor, |
223 | 64 | TestCase, | 74 | TestCase, |
224 | @@ -93,39 +103,54 @@ | |||
225 | 93 | return os.path.join(self.root, 'distributionmirror_http_server.log') | 103 | return os.path.join(self.root, 'distributionmirror_http_server.log') |
226 | 94 | 104 | ||
227 | 95 | 105 | ||
229 | 96 | class TestProberProtocolAndFactory(TrialTestCase): | 106 | class TestProberProtocolAndFactory(TestCase): |
230 | 97 | 107 | ||
231 | 98 | layer = TwistedLayer | 108 | layer = TwistedLayer |
232 | 109 | run_tests_with = AsynchronousDeferredRunTestForBrokenTwisted.make_factory( | ||
233 | 110 | timeout=30) | ||
234 | 99 | 111 | ||
235 | 100 | def setUp(self): | 112 | def setUp(self): |
237 | 101 | self.orig_proxy = os.getenv('http_proxy') | 113 | super(TestProberProtocolAndFactory, self).setUp() |
238 | 102 | root = DistributionMirrorTestHTTPServer() | 114 | root = DistributionMirrorTestHTTPServer() |
239 | 103 | site = server.Site(root) | 115 | site = server.Site(root) |
240 | 104 | site.displayTracebacks = False | 116 | site.displayTracebacks = False |
241 | 105 | self.listening_port = reactor.listenTCP(0, site) | 117 | self.listening_port = reactor.listenTCP(0, site) |
242 | 118 | self.addCleanup(self.listening_port.stopListening) | ||
243 | 106 | self.port = self.listening_port.getHost().port | 119 | self.port = self.listening_port.getHost().port |
244 | 107 | self.urls = {'timeout': u'http://localhost:%s/timeout' % self.port, | 120 | self.urls = {'timeout': u'http://localhost:%s/timeout' % self.port, |
245 | 108 | '200': u'http://localhost:%s/valid-mirror' % self.port, | 121 | '200': u'http://localhost:%s/valid-mirror' % self.port, |
246 | 109 | '500': u'http://localhost:%s/error' % self.port, | 122 | '500': u'http://localhost:%s/error' % self.port, |
247 | 110 | '404': u'http://localhost:%s/invalid-mirror' % self.port} | 123 | '404': u'http://localhost:%s/invalid-mirror' % self.port} |
252 | 111 | 124 | self.pushConfig('launchpad', http_proxy=None) | |
249 | 112 | def tearDown(self): | ||
250 | 113 | restore_http_proxy(self.orig_proxy) | ||
251 | 114 | return self.listening_port.stopListening() | ||
253 | 115 | 125 | ||
254 | 116 | def _createProberAndProbe(self, url): | 126 | def _createProberAndProbe(self, url): |
255 | 117 | prober = ProberFactory(url) | 127 | prober = ProberFactory(url) |
256 | 118 | return prober.probe() | 128 | return prober.probe() |
257 | 119 | 129 | ||
267 | 120 | def test_environment_http_proxy_is_handled_correctly(self): | 130 | def test_config_no_http_proxy(self): |
268 | 121 | os.environ['http_proxy'] = 'http://squid.internal:3128' | 131 | prober = ProberFactory(self.urls['200']) |
269 | 122 | prober = ProberFactory(self.urls['200']) | 132 | self.assertThat(prober, MatchesStructure.byEquality( |
270 | 123 | self.assertEqual(prober.request_host, 'localhost') | 133 | request_scheme='http', |
271 | 124 | self.assertEqual(prober.request_port, self.port) | 134 | request_host='localhost', |
272 | 125 | self.assertEqual(prober.request_path, '/valid-mirror') | 135 | request_port=self.port, |
273 | 126 | self.assertEqual(prober.connect_host, 'squid.internal') | 136 | request_path='/valid-mirror', |
274 | 127 | self.assertEqual(prober.connect_port, 3128) | 137 | connect_scheme='http', |
275 | 128 | self.assertEqual(prober.connect_path, self.urls['200']) | 138 | connect_host='localhost', |
276 | 139 | connect_port=self.port, | ||
277 | 140 | connect_path='/valid-mirror')) | ||
278 | 141 | |||
279 | 142 | def test_config_http_proxy(self): | ||
280 | 143 | self.pushConfig('launchpad', http_proxy='http://squid.internal:3128') | ||
281 | 144 | prober = ProberFactory(self.urls['200']) | ||
282 | 145 | self.assertThat(prober, MatchesStructure.byEquality( | ||
283 | 146 | request_scheme='http', | ||
284 | 147 | request_host='localhost', | ||
285 | 148 | request_port=self.port, | ||
286 | 149 | request_path='/valid-mirror', | ||
287 | 150 | connect_scheme='http', | ||
288 | 151 | connect_host='squid.internal', | ||
289 | 152 | connect_port=3128, | ||
290 | 153 | connect_path=self.urls['200'])) | ||
291 | 129 | 154 | ||
292 | 130 | def test_connect_cancels_existing_timeout_call(self): | 155 | def test_connect_cancels_existing_timeout_call(self): |
293 | 131 | prober = ProberFactory(self.urls['200']) | 156 | prober = ProberFactory(self.urls['200']) |
294 | @@ -161,14 +186,10 @@ | |||
295 | 161 | return deferred.addCallback(restore_connect, orig_connect) | 186 | return deferred.addCallback(restore_connect, orig_connect) |
296 | 162 | 187 | ||
297 | 163 | def test_connect_to_proxy_when_http_proxy_exists(self): | 188 | def test_connect_to_proxy_when_http_proxy_exists(self): |
299 | 164 | os.environ['http_proxy'] = 'http://squid.internal:3128' | 189 | self.pushConfig('launchpad', http_proxy='http://squid.internal:3128') |
300 | 165 | self._test_connect_to_host(self.urls['200'], 'squid.internal') | 190 | self._test_connect_to_host(self.urls['200'], 'squid.internal') |
301 | 166 | 191 | ||
302 | 167 | def test_connect_to_host_when_http_proxy_does_not_exist(self): | 192 | def test_connect_to_host_when_http_proxy_does_not_exist(self): |
303 | 168 | try: | ||
304 | 169 | del os.environ['http_proxy'] | ||
305 | 170 | except KeyError: | ||
306 | 171 | pass | ||
307 | 172 | self._test_connect_to_host(self.urls['200'], 'localhost') | 193 | self._test_connect_to_host(self.urls['200'], 'localhost') |
308 | 173 | 194 | ||
309 | 174 | def test_probe_sets_up_timeout_call(self): | 195 | def test_probe_sets_up_timeout_call(self): |
310 | @@ -197,13 +218,13 @@ | |||
311 | 197 | prober = RedirectAwareProberFactory( | 218 | prober = RedirectAwareProberFactory( |
312 | 198 | 'http://localhost:%s/redirect-infinite-loop' % self.port) | 219 | 'http://localhost:%s/redirect-infinite-loop' % self.port) |
313 | 199 | deferred = prober.probe() | 220 | deferred = prober.probe() |
315 | 200 | return self.assertFailure(deferred, InfiniteLoopDetected) | 221 | return assert_fails_with(deferred, InfiniteLoopDetected) |
316 | 201 | 222 | ||
317 | 202 | def test_redirectawareprober_fail_on_unknown_scheme(self): | 223 | def test_redirectawareprober_fail_on_unknown_scheme(self): |
318 | 203 | prober = RedirectAwareProberFactory( | 224 | prober = RedirectAwareProberFactory( |
319 | 204 | 'http://localhost:%s/redirect-unknown-url-scheme' % self.port) | 225 | 'http://localhost:%s/redirect-unknown-url-scheme' % self.port) |
320 | 205 | deferred = prober.probe() | 226 | deferred = prober.probe() |
322 | 206 | return self.assertFailure(deferred, UnknownURLSchemeAfterRedirect) | 227 | return assert_fails_with(deferred, UnknownURLSchemeAfterRedirect) |
323 | 207 | 228 | ||
324 | 208 | def test_200(self): | 229 | def test_200(self): |
325 | 209 | d = self._createProberAndProbe(self.urls['200']) | 230 | d = self._createProberAndProbe(self.urls['200']) |
326 | @@ -237,15 +258,15 @@ | |||
327 | 237 | 258 | ||
328 | 238 | def test_notfound(self): | 259 | def test_notfound(self): |
329 | 239 | d = self._createProberAndProbe(self.urls['404']) | 260 | d = self._createProberAndProbe(self.urls['404']) |
331 | 240 | return self.assertFailure(d, BadResponseCode) | 261 | return assert_fails_with(d, BadResponseCode) |
332 | 241 | 262 | ||
333 | 242 | def test_500(self): | 263 | def test_500(self): |
334 | 243 | d = self._createProberAndProbe(self.urls['500']) | 264 | d = self._createProberAndProbe(self.urls['500']) |
336 | 244 | return self.assertFailure(d, BadResponseCode) | 265 | return assert_fails_with(d, BadResponseCode) |
337 | 245 | 266 | ||
338 | 246 | def test_timeout(self): | 267 | def test_timeout(self): |
339 | 247 | d = self._createProberAndProbe(self.urls['timeout']) | 268 | d = self._createProberAndProbe(self.urls['timeout']) |
341 | 248 | return self.assertFailure(d, ProberTimeout) | 269 | return assert_fails_with(d, ProberTimeout) |
342 | 249 | 270 | ||
343 | 250 | def test_prober_user_agent(self): | 271 | def test_prober_user_agent(self): |
344 | 251 | protocol = RedirectAwareProberProtocol() | 272 | protocol = RedirectAwareProberProtocol() |
345 | @@ -380,7 +401,7 @@ | |||
346 | 380 | self.assertFalse(should_skip_host(self.host)) | 401 | self.assertFalse(should_skip_host(self.host)) |
347 | 381 | 402 | ||
348 | 382 | 403 | ||
350 | 383 | class TestProberFactoryRequestTimeoutRatioWithTwisted(TrialTestCase): | 404 | class TestProberFactoryRequestTimeoutRatioWithTwisted(TestCase): |
351 | 384 | """Tests to ensure we stop issuing requests on a given host if the | 405 | """Tests to ensure we stop issuing requests on a given host if the |
352 | 385 | requests/timeouts ratio on that host is too low. | 406 | requests/timeouts ratio on that host is too low. |
353 | 386 | 407 | ||
354 | @@ -392,25 +413,29 @@ | |||
355 | 392 | """ | 413 | """ |
356 | 393 | 414 | ||
357 | 394 | layer = TwistedLayer | 415 | layer = TwistedLayer |
358 | 416 | run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=30) | ||
359 | 395 | 417 | ||
360 | 396 | def setUp(self): | 418 | def setUp(self): |
365 | 397 | self.orig_host_requests = dict( | 419 | super(TestProberFactoryRequestTimeoutRatioWithTwisted, self).setUp() |
366 | 398 | distributionmirror_prober.host_requests) | 420 | orig_host_requests = dict(distributionmirror_prober.host_requests) |
367 | 399 | self.orig_host_timeouts = dict( | 421 | orig_host_timeouts = dict(distributionmirror_prober.host_timeouts) |
364 | 400 | distributionmirror_prober.host_timeouts) | ||
368 | 401 | distributionmirror_prober.host_requests = {} | 422 | distributionmirror_prober.host_requests = {} |
369 | 402 | distributionmirror_prober.host_timeouts = {} | 423 | distributionmirror_prober.host_timeouts = {} |
370 | 424 | |||
371 | 425 | def restore_prober_globals(): | ||
372 | 426 | # Restore the globals that our tests fiddle with. | ||
373 | 427 | distributionmirror_prober.host_requests = orig_host_requests | ||
374 | 428 | distributionmirror_prober.host_timeouts = orig_host_timeouts | ||
375 | 429 | |||
376 | 430 | self.addCleanup(restore_prober_globals) | ||
377 | 431 | |||
378 | 403 | root = DistributionMirrorTestHTTPServer() | 432 | root = DistributionMirrorTestHTTPServer() |
379 | 404 | site = server.Site(root) | 433 | site = server.Site(root) |
380 | 405 | site.displayTracebacks = False | 434 | site.displayTracebacks = False |
381 | 406 | self.listening_port = reactor.listenTCP(0, site) | 435 | self.listening_port = reactor.listenTCP(0, site) |
382 | 436 | self.addCleanup(self.listening_port.stopListening) | ||
383 | 407 | self.port = self.listening_port.getHost().port | 437 | self.port = self.listening_port.getHost().port |
390 | 408 | 438 | self.pushConfig('launchpad', http_proxy=None) | |
385 | 409 | def tearDown(self): | ||
386 | 410 | # Restore the globals that our tests fiddle with. | ||
387 | 411 | distributionmirror_prober.host_requests = self.orig_host_requests | ||
388 | 412 | distributionmirror_prober.host_timeouts = self.orig_host_timeouts | ||
389 | 413 | return self.listening_port.stopListening() | ||
391 | 414 | 439 | ||
392 | 415 | def _createProberAndProbe(self, url): | 440 | def _createProberAndProbe(self, url): |
393 | 416 | prober = ProberFactory(url) | 441 | prober = ProberFactory(url) |
394 | @@ -455,7 +480,7 @@ | |||
395 | 455 | 480 | ||
396 | 456 | d = self._createProberAndProbe( | 481 | d = self._createProberAndProbe( |
397 | 457 | u'http://%s:%s/timeout' % (host, self.port)) | 482 | u'http://%s:%s/timeout' % (host, self.port)) |
399 | 458 | return self.assertFailure(d, ConnectionSkipped) | 483 | return assert_fails_with(d, ConnectionSkipped) |
400 | 459 | 484 | ||
401 | 460 | 485 | ||
402 | 461 | class TestMultiLock(TestCase): | 486 | class TestMultiLock(TestCase): |
403 | @@ -842,7 +867,8 @@ | |||
404 | 842 | self.assertNotEqual(0, len(mirror.cdimage_series)) | 867 | self.assertNotEqual(0, len(mirror.cdimage_series)) |
405 | 843 | # Note that calling this function won't actually probe any mirrors; we | 868 | # Note that calling this function won't actually probe any mirrors; we |
406 | 844 | # need to call reactor.run() to actually start the probing. | 869 | # need to call reactor.run() to actually start the probing. |
408 | 845 | probe_cdimage_mirror(mirror, StringIO(), [], logging) | 870 | with default_timeout(15.0): |
409 | 871 | probe_cdimage_mirror(mirror, StringIO(), [], logging) | ||
410 | 846 | self.assertEqual(0, len(mirror.cdimage_series)) | 872 | self.assertEqual(0, len(mirror.cdimage_series)) |
411 | 847 | 873 | ||
412 | 848 | def test_archive_mirror_probe_function(self): | 874 | def test_archive_mirror_probe_function(self): |
413 | @@ -856,8 +882,9 @@ | |||
414 | 856 | mirror1 = DistributionMirror.byName('releases-mirror') | 882 | mirror1 = DistributionMirror.byName('releases-mirror') |
415 | 857 | mirror2 = DistributionMirror.byName('releases-mirror2') | 883 | mirror2 = DistributionMirror.byName('releases-mirror2') |
416 | 858 | mirror3 = DistributionMirror.byName('canonical-releases') | 884 | mirror3 = DistributionMirror.byName('canonical-releases') |
419 | 859 | self._test_one_semaphore_for_each_host( | 885 | with default_timeout(15.0): |
420 | 860 | mirror1, mirror2, mirror3, probe_cdimage_mirror) | 886 | self._test_one_semaphore_for_each_host( |
421 | 887 | mirror1, mirror2, mirror3, probe_cdimage_mirror) | ||
422 | 861 | 888 | ||
423 | 862 | def _test_one_semaphore_for_each_host( | 889 | def _test_one_semaphore_for_each_host( |
424 | 863 | self, mirror1, mirror2, mirror3, probe_function): | 890 | self, mirror1, mirror2, mirror3, probe_function): |
425 | @@ -905,20 +932,24 @@ | |||
426 | 905 | # When using an http_proxy, even though we'll actually connect to the | 932 | # When using an http_proxy, even though we'll actually connect to the |
427 | 906 | # proxy, we'll use the mirror's host as the key to find the semaphore | 933 | # proxy, we'll use the mirror's host as the key to find the semaphore |
428 | 907 | # that should be used | 934 | # that should be used |
431 | 908 | orig_proxy = os.getenv('http_proxy') | 935 | self.pushConfig('launchpad', http_proxy='http://squid.internal:3128/') |
430 | 909 | os.environ['http_proxy'] = 'http://squid.internal:3128/' | ||
432 | 910 | probe_function(mirror3, StringIO(), [], logging) | 936 | probe_function(mirror3, StringIO(), [], logging) |
433 | 911 | self.assertEqual(len(request_manager.host_locks), 2) | 937 | self.assertEqual(len(request_manager.host_locks), 2) |
434 | 912 | restore_http_proxy(orig_proxy) | ||
435 | 913 | 938 | ||
436 | 914 | 939 | ||
437 | 915 | class TestCDImageFileListFetching(TestCase): | 940 | class TestCDImageFileListFetching(TestCase): |
438 | 916 | 941 | ||
439 | 942 | @responses.activate | ||
440 | 917 | def test_no_cache(self): | 943 | def test_no_cache(self): |
441 | 918 | url = 'http://releases.ubuntu.com/.manifest' | 944 | url = 'http://releases.ubuntu.com/.manifest' |
445 | 919 | request = _build_request_for_cdimage_file_list(url) | 945 | self.pushConfig('distributionmirrorprober', cdimage_file_list_url=url) |
446 | 920 | self.assertEqual(request.headers['Pragma'], 'no-cache') | 946 | responses.add('GET', url) |
447 | 921 | self.assertEqual(request.headers['Cache-control'], 'no-cache') | 947 | with default_timeout(1.0): |
448 | 948 | _get_cdimage_file_list() | ||
449 | 949 | self.assertThat(responses.calls[0].request.headers, ContainsDict({ | ||
450 | 950 | 'Pragma': Equals('no-cache'), | ||
451 | 951 | 'Cache-control': Equals('no-cache'), | ||
452 | 952 | })) | ||
453 | 922 | 953 | ||
454 | 923 | 954 | ||
455 | 924 | class TestLoggingMixin(TestCase): | 955 | class TestLoggingMixin(TestCase): |
456 | 925 | 956 | ||
457 | === modified file 'lib/lp/services/tests/test_timeout.py' | |||
458 | --- lib/lp/services/tests/test_timeout.py 2018-06-26 19:17:19 +0000 | |||
459 | +++ lib/lp/services/tests/test_timeout.py 2018-07-02 11:30:55 +0000 | |||
460 | @@ -367,26 +367,6 @@ | |||
461 | 367 | MatchesStructure.byEquality(status_code=200, content='Success.')) | 367 | MatchesStructure.byEquality(status_code=200, content='Success.')) |
462 | 368 | t.join() | 368 | t.join() |
463 | 369 | 369 | ||
464 | 370 | def test_urlfetch_does_not_support_ftp_urls(self): | ||
465 | 371 | """urlfetch() does not support ftp urls.""" | ||
466 | 372 | set_default_timeout_function(lambda: 1) | ||
467 | 373 | self.addCleanup(set_default_timeout_function, None) | ||
468 | 374 | url = 'ftp://localhost/' | ||
469 | 375 | e = self.assertRaises(InvalidSchema, urlfetch, url) | ||
470 | 376 | self.assertEqual( | ||
471 | 377 | "No connection adapters were found for '%s'" % url, str(e)) | ||
472 | 378 | |||
473 | 379 | def test_urlfetch_does_not_support_file_urls_by_default(self): | ||
474 | 380 | """urlfetch() does not support file urls by default.""" | ||
475 | 381 | set_default_timeout_function(lambda: 1) | ||
476 | 382 | self.addCleanup(set_default_timeout_function, None) | ||
477 | 383 | test_path = self.useFixture(TempDir()).join('file') | ||
478 | 384 | write_file(test_path, '') | ||
479 | 385 | url = 'file://' + test_path | ||
480 | 386 | e = self.assertRaises(InvalidSchema, urlfetch, url) | ||
481 | 387 | self.assertEqual( | ||
482 | 388 | "No connection adapters were found for '%s'" % url, str(e)) | ||
483 | 389 | |||
484 | 390 | def test_urlfetch_no_proxy_by_default(self): | 370 | def test_urlfetch_no_proxy_by_default(self): |
485 | 391 | """urlfetch does not use a proxy by default.""" | 371 | """urlfetch does not use a proxy by default.""" |
486 | 392 | self.pushConfig('launchpad', http_proxy='http://proxy.example:3128/') | 372 | self.pushConfig('launchpad', http_proxy='http://proxy.example:3128/') |
487 | @@ -418,6 +398,56 @@ | |||
488 | 418 | {scheme: proxy for scheme in ('http', 'https')}, | 398 | {scheme: proxy for scheme in ('http', 'https')}, |
489 | 419 | fake_send.calls[0][1]['proxies']) | 399 | fake_send.calls[0][1]['proxies']) |
490 | 420 | 400 | ||
491 | 401 | def test_urlfetch_does_not_support_ftp_urls_by_default(self): | ||
492 | 402 | """urlfetch() does not support ftp urls by default.""" | ||
493 | 403 | set_default_timeout_function(lambda: 1) | ||
494 | 404 | self.addCleanup(set_default_timeout_function, None) | ||
495 | 405 | url = 'ftp://localhost/' | ||
496 | 406 | e = self.assertRaises(InvalidSchema, urlfetch, url) | ||
497 | 407 | self.assertEqual( | ||
498 | 408 | "No connection adapters were found for '%s'" % url, str(e)) | ||
499 | 409 | |||
500 | 410 | def test_urlfetch_supports_ftp_urls_if_allow_ftp(self): | ||
501 | 411 | """urlfetch() supports ftp urls via a proxy if explicitly asked.""" | ||
502 | 412 | sock, http_server_url = self.make_test_socket() | ||
503 | 413 | sock.listen(1) | ||
504 | 414 | |||
505 | 415 | def success_result(): | ||
506 | 416 | (client_sock, client_addr) = sock.accept() | ||
507 | 417 | # We only provide a test HTTP proxy, not anything beyond that. | ||
508 | 418 | client_sock.sendall(dedent("""\ | ||
509 | 419 | HTTP/1.0 200 Ok | ||
510 | 420 | Content-Type: text/plain | ||
511 | 421 | Content-Length: 8 | ||
512 | 422 | |||
513 | 423 | Success.""")) | ||
514 | 424 | client_sock.close() | ||
515 | 425 | |||
516 | 426 | self.pushConfig('launchpad', http_proxy=http_server_url) | ||
517 | 427 | t = threading.Thread(target=success_result) | ||
518 | 428 | t.start() | ||
519 | 429 | set_default_timeout_function(lambda: 1) | ||
520 | 430 | self.addCleanup(set_default_timeout_function, None) | ||
521 | 431 | response = urlfetch( | ||
522 | 432 | 'ftp://example.com/', trust_env=False, use_proxy=True, | ||
523 | 433 | allow_ftp=True) | ||
524 | 434 | self.assertThat(response, MatchesStructure( | ||
525 | 435 | status_code=Equals(200), | ||
526 | 436 | headers=ContainsDict({'content-length': Equals('8')}), | ||
527 | 437 | content=Equals('Success.'))) | ||
528 | 438 | t.join() | ||
529 | 439 | |||
530 | 440 | def test_urlfetch_does_not_support_file_urls_by_default(self): | ||
531 | 441 | """urlfetch() does not support file urls by default.""" | ||
532 | 442 | set_default_timeout_function(lambda: 1) | ||
533 | 443 | self.addCleanup(set_default_timeout_function, None) | ||
534 | 444 | test_path = self.useFixture(TempDir()).join('file') | ||
535 | 445 | write_file(test_path, '') | ||
536 | 446 | url = 'file://' + test_path | ||
537 | 447 | e = self.assertRaises(InvalidSchema, urlfetch, url) | ||
538 | 448 | self.assertEqual( | ||
539 | 449 | "No connection adapters were found for '%s'" % url, str(e)) | ||
540 | 450 | |||
541 | 421 | def test_urlfetch_supports_file_urls_if_allow_file(self): | 451 | def test_urlfetch_supports_file_urls_if_allow_file(self): |
542 | 422 | """urlfetch() supports file urls if explicitly asked to do so.""" | 452 | """urlfetch() supports file urls if explicitly asked to do so.""" |
543 | 423 | set_default_timeout_function(lambda: 1) | 453 | set_default_timeout_function(lambda: 1) |
544 | 424 | 454 | ||
545 | === modified file 'lib/lp/services/timeout.py' | |||
546 | --- lib/lp/services/timeout.py 2018-06-26 19:17:19 +0000 | |||
547 | +++ lib/lp/services/timeout.py 2018-07-02 11:30:55 +0000 | |||
548 | @@ -324,8 +324,8 @@ | |||
549 | 324 | self.session = None | 324 | self.session = None |
550 | 325 | 325 | ||
551 | 326 | @with_timeout(cleanup='cleanup') | 326 | @with_timeout(cleanup='cleanup') |
554 | 327 | def fetch(self, url, trust_env=None, use_proxy=False, allow_file=False, | 327 | def fetch(self, url, trust_env=None, use_proxy=False, allow_ftp=False, |
555 | 328 | output_file=None, **request_kwargs): | 328 | allow_file=False, output_file=None, **request_kwargs): |
556 | 329 | """Fetch the URL using a custom HTTP handler supporting timeout. | 329 | """Fetch the URL using a custom HTTP handler supporting timeout. |
557 | 330 | 330 | ||
558 | 331 | :param url: The URL to fetch. | 331 | :param url: The URL to fetch. |
559 | @@ -333,6 +333,7 @@ | |||
560 | 333 | to determine whether it fetches proxy configuration from the | 333 | to determine whether it fetches proxy configuration from the |
561 | 334 | environment. | 334 | environment. |
562 | 335 | :param use_proxy: If True, use Launchpad's configured proxy. | 335 | :param use_proxy: If True, use Launchpad's configured proxy. |
563 | 336 | :param allow_ftp: If True, allow ftp:// URLs. | ||
564 | 336 | :param allow_file: If True, allow file:// URLs. (Be careful to only | 337 | :param allow_file: If True, allow file:// URLs. (Be careful to only |
565 | 337 | pass this if the URL is trusted.) | 338 | pass this if the URL is trusted.) |
566 | 338 | :param output_file: If not None, download the response content to | 339 | :param output_file: If not None, download the response content to |
567 | @@ -346,6 +347,9 @@ | |||
568 | 346 | # Mount our custom adapters. | 347 | # Mount our custom adapters. |
569 | 347 | self.session.mount("https://", CleanableHTTPAdapter()) | 348 | self.session.mount("https://", CleanableHTTPAdapter()) |
570 | 348 | self.session.mount("http://", CleanableHTTPAdapter()) | 349 | self.session.mount("http://", CleanableHTTPAdapter()) |
571 | 350 | # We can do FTP, but currently only via an HTTP proxy. | ||
572 | 351 | if allow_ftp and use_proxy: | ||
573 | 352 | self.session.mount("ftp://", CleanableHTTPAdapter()) | ||
574 | 349 | if allow_file: | 353 | if allow_file: |
575 | 350 | self.session.mount("file://", FileAdapter()) | 354 | self.session.mount("file://", FileAdapter()) |
576 | 351 | 355 | ||
577 | @@ -354,6 +358,8 @@ | |||
578 | 354 | request_kwargs.setdefault("proxies", {}) | 358 | request_kwargs.setdefault("proxies", {}) |
579 | 355 | request_kwargs["proxies"]["http"] = config.launchpad.http_proxy | 359 | request_kwargs["proxies"]["http"] = config.launchpad.http_proxy |
580 | 356 | request_kwargs["proxies"]["https"] = config.launchpad.http_proxy | 360 | request_kwargs["proxies"]["https"] = config.launchpad.http_proxy |
581 | 361 | if allow_ftp: | ||
582 | 362 | request_kwargs["proxies"]["ftp"] = config.launchpad.http_proxy | ||
583 | 357 | if output_file is not None: | 363 | if output_file is not None: |
584 | 358 | request_kwargs["stream"] = True | 364 | request_kwargs["stream"] = True |
585 | 359 | response = self.session.request(url=url, **request_kwargs) | 365 | response = self.session.request(url=url, **request_kwargs) |