Merge ~pappacena/launchpad:https-mirrors-2 into launchpad:master
- Git
- lp:~pappacena/launchpad
- https-mirrors-2
- Merge into master
Proposed by
Thiago F. Pappacena
Status: | Merged | ||||
---|---|---|---|---|---|
Approved by: | Thiago F. Pappacena | ||||
Approved revision: | 7448ccd84ef29a536bac827ece5b8116ae73664a | ||||
Merge reported by: | Otto Co-Pilot | ||||
Merged at revision: | not available | ||||
Proposed branch: | ~pappacena/launchpad:https-mirrors-2 | ||||
Merge into: | launchpad:master | ||||
Diff against target: |
1116 lines (+477/-53) 16 files modified
lib/lp/registry/browser/distributionmirror.py (+7/-6) lib/lp/registry/browser/tests/distributionmirror-views.txt (+40/-9) lib/lp/registry/configure.zcml (+5/-3) lib/lp/registry/interfaces/distribution.py (+4/-4) lib/lp/registry/interfaces/distributionmirror.py (+21/-3) lib/lp/registry/model/distribution.py (+7/-4) lib/lp/registry/model/distributionmirror.py (+10/-2) lib/lp/registry/scripts/distributionmirror_prober.py (+153/-7) lib/lp/registry/stories/webservice/xx-distribution-mirror.txt (+4/-0) lib/lp/registry/stories/webservice/xx-distribution.txt (+1/-0) lib/lp/registry/templates/distributionmirror-index.pt (+4/-0) lib/lp/registry/templates/distributionmirror-macros.pt (+3/-1) lib/lp/registry/tests/distributionmirror_http_server.py (+15/-7) lib/lp/registry/tests/test_distributionmirror_prober.py (+196/-3) lib/lp/scripts/utilities/importpedant.py (+3/-1) lib/lp/testing/factory.py (+4/-3) |
||||
Related bugs: |
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Colin Watson (community) | Approve | ||
Review via email: mp+379913@code.launchpad.net |
Commit message
Undoing git revert on: Allowing to register HTTPS mirrors (both for CD images and archives)
Description of the change
This MP is undoing the git revert of HTTPS mirrors feature on master (https:/
To post a comment you must log in.
Revision history for this message
Thiago F. Pappacena (pappacena) wrote : | # |
Great! Thanks, cjwatson!
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | diff --git a/lib/lp/registry/browser/distributionmirror.py b/lib/lp/registry/browser/distributionmirror.py |
2 | index fd671a0..6ae1eb7 100644 |
3 | --- a/lib/lp/registry/browser/distributionmirror.py |
4 | +++ b/lib/lp/registry/browser/distributionmirror.py |
5 | @@ -1,4 +1,4 @@ |
6 | -# Copyright 2009 Canonical Ltd. This software is licensed under the |
7 | +# Copyright 2009-2020 Canonical Ltd. This software is licensed under the |
8 | # GNU Affero General Public License version 3 (see the file LICENSE). |
9 | |
10 | __metaclass__ = type |
11 | @@ -207,9 +207,9 @@ class DistributionMirrorDeleteView(LaunchpadFormView): |
12 | class DistributionMirrorAddView(LaunchpadFormView): |
13 | schema = IDistributionMirror |
14 | field_names = [ |
15 | - "display_name", "description", "whiteboard", "http_base_url", |
16 | - "ftp_base_url", "rsync_base_url", "speed", "country", "content", |
17 | - "official_candidate", |
18 | + "display_name", "description", "whiteboard", "https_base_url", |
19 | + "http_base_url", "ftp_base_url", "rsync_base_url", "speed", "country", |
20 | + "content", "official_candidate", |
21 | ] |
22 | invariant_context = None |
23 | |
24 | @@ -235,6 +235,7 @@ class DistributionMirrorAddView(LaunchpadFormView): |
25 | content=data['content'], display_name=data['display_name'], |
26 | description=data['description'], |
27 | whiteboard=data['whiteboard'], |
28 | + https_base_url=data['https_base_url'], |
29 | http_base_url=data['http_base_url'], |
30 | ftp_base_url=data['ftp_base_url'], |
31 | rsync_base_url=data['rsync_base_url'], |
32 | @@ -279,8 +280,8 @@ class DistributionMirrorEditView(LaunchpadEditFormView): |
33 | schema = IDistributionMirror |
34 | field_names = [ |
35 | "name", "display_name", "description", "whiteboard", |
36 | - "http_base_url", "ftp_base_url", "rsync_base_url", "speed", |
37 | - "country", "content", "official_candidate", |
38 | + "https_base_url", "http_base_url", "ftp_base_url", "rsync_base_url", |
39 | + "speed", "country", "content", "official_candidate", |
40 | ] |
41 | |
42 | @property |
43 | diff --git a/lib/lp/registry/browser/tests/distributionmirror-views.txt b/lib/lp/registry/browser/tests/distributionmirror-views.txt |
44 | index e11dbd9..f8fe4c5 100644 |
45 | --- a/lib/lp/registry/browser/tests/distributionmirror-views.txt |
46 | +++ b/lib/lp/registry/browser/tests/distributionmirror-views.txt |
47 | @@ -44,18 +44,19 @@ The view provides a label, page_title, and cancel_url |
48 | >>> print view.cancel_url |
49 | http://launchpad.test/ubuntu |
50 | |
51 | -A HTTP or FTP URL is required to register a mirror. |
52 | +A HTTP, HTTPS or FTP URL is required to register a mirror. |
53 | |
54 | >>> view.field_names |
55 | - ['display_name', 'description', 'whiteboard', 'http_base_url', |
56 | - 'ftp_base_url', 'rsync_base_url', 'speed', 'country', 'content', |
57 | - 'official_candidate'] |
58 | + ['display_name', 'description', 'whiteboard', 'https_base_url', |
59 | + 'http_base_url', 'ftp_base_url', 'rsync_base_url', 'speed', 'country', |
60 | + 'content', 'official_candidate'] |
61 | |
62 | >>> form = { |
63 | ... 'field.display_name': 'Illuminati', |
64 | ... 'field.description': 'description', |
65 | ... 'field.whiteboard': 'whiteboard', |
66 | ... 'field.http_base_url': 'http://secret.me/', |
67 | + ... 'field.https_base_url': '', |
68 | ... 'field.ftp_base_url': '', |
69 | ... 'field.rsync_base_url': '', |
70 | ... 'field.speed': 'S128K', |
71 | @@ -93,6 +94,7 @@ not significant). |
72 | The same is true for a FTP URL. |
73 | |
74 | >>> mirror.ftp_base_url = 'ftp://now-here.me/' |
75 | + >>> bad_form['field.https_base_url'] = '' |
76 | >>> bad_form['field.http_base_url'] = '' |
77 | >>> bad_form['field.ftp_base_url'] = 'ftp://now-here.me' |
78 | >>> view = create_initialized_view(ubuntu, '+newmirror', form=bad_form) |
79 | @@ -110,14 +112,15 @@ The same is true for a rsync URL. |
80 | ... print error[2] |
81 | The distribution mirror ... is already registered with this URL. |
82 | |
83 | -A mirror must have an ftp or http URL. |
84 | +A mirror must have an ftp, HTTPS or http URL. |
85 | |
86 | + >>> bad_form['field.https_base_url'] = '' |
87 | >>> bad_form['field.http_base_url'] = '' |
88 | >>> bad_form['field.ftp_base_url'] = '' |
89 | >>> view = create_initialized_view(ubuntu, '+newmirror', form=bad_form) |
90 | >>> for message in view.errors: |
91 | ... print message |
92 | - A mirror must have at least an HTTP or FTP URL. |
93 | + A mirror must have at least an HTTP(S) or FTP URL. |
94 | |
95 | The URL cannot contain a fragment. |
96 | |
97 | @@ -135,6 +138,34 @@ The URL cannot contain a query string. |
98 | ... print error[2] |
99 | URIs with query strings are not allowed. |
100 | |
101 | +The HTTPS URL may not have an HTTP scheme. |
102 | + |
103 | + >>> bad_form['field.http_base_url'] = '' |
104 | + >>> bad_form['field.https_base_url'] = 'http://secret.me/#fragement' |
105 | + >>> view = create_initialized_view(ubuntu, '+newmirror', form=bad_form) |
106 | + >>> for error in view.errors: |
107 | + ... print error[2] |
108 | + The URI scheme "http" is not allowed. |
109 | + Only URIs with the following schemes may be used: https |
110 | + |
111 | +The HTTPS URL cannot contain a fragment. |
112 | + |
113 | + >>> bad_form['field.http_base_url'] = '' |
114 | + >>> bad_form['field.https_base_url'] = 'https://secret.me/#fragement' |
115 | + >>> view = create_initialized_view(ubuntu, '+newmirror', form=bad_form) |
116 | + >>> for error in view.errors: |
117 | + ... print error[2] |
118 | + URIs with fragment identifiers are not allowed. |
119 | + |
120 | +The URL cannot contain a query string. |
121 | + |
122 | + >>> bad_form['field.http_base_url'] = '' |
123 | + >>> bad_form['field.https_base_url'] = 'https://secret.me/?query=string' |
124 | + >>> view = create_initialized_view(ubuntu, '+newmirror', form=bad_form) |
125 | + >>> for error in view.errors: |
126 | + ... print error[2] |
127 | + URIs with query strings are not allowed. |
128 | + |
129 | |
130 | Reviewing a distribution mirror |
131 | ------------------------------- |
132 | @@ -239,9 +270,9 @@ The +edit view provides a label, page_title, and cancel_url. |
133 | The user can edit the mirror fields. |
134 | |
135 | >>> view.field_names |
136 | - ['name', 'display_name', 'description', 'whiteboard', 'http_base_url', |
137 | - 'ftp_base_url', 'rsync_base_url', 'speed', 'country', 'content', |
138 | - 'official_candidate'] |
139 | + ['name', 'display_name', 'description', 'whiteboard', 'https_base_url', |
140 | + 'http_base_url', 'ftp_base_url', 'rsync_base_url', 'speed', 'country', |
141 | + 'content', 'official_candidate'] |
142 | |
143 | >>> print mirror.ftp_base_url |
144 | None |
145 | diff --git a/lib/lp/registry/configure.zcml b/lib/lp/registry/configure.zcml |
146 | index 9be68e0..d5875c5 100644 |
147 | --- a/lib/lp/registry/configure.zcml |
148 | +++ b/lib/lp/registry/configure.zcml |
149 | @@ -1,4 +1,4 @@ |
150 | -<!-- Copyright 2009-2019 Canonical Ltd. This software is licensed under the |
151 | +<!-- Copyright 2009-2020 Canonical Ltd. This software is licensed under the |
152 | GNU Affero General Public License version 3 (see the file LICENSE). |
153 | --> |
154 | |
155 | @@ -2046,6 +2046,7 @@ |
156 | description |
157 | distribution |
158 | http_base_url |
159 | + https_base_url |
160 | ftp_base_url |
161 | rsync_base_url |
162 | enabled |
163 | @@ -2084,8 +2085,9 @@ |
164 | <require |
165 | permission="launchpad.Edit" |
166 | set_attributes="name display_name description whiteboard |
167 | - http_base_url ftp_base_url rsync_base_url enabled |
168 | - speed country content official_candidate owner" |
169 | + http_base_url https_base_url ftp_base_url |
170 | + rsync_base_url enabled speed country content |
171 | + official_candidate owner" |
172 | attributes="official_candidate whiteboard resubmitForReview" /> |
173 | <require |
174 | permission="launchpad.Admin" |
175 | diff --git a/lib/lp/registry/interfaces/distribution.py b/lib/lp/registry/interfaces/distribution.py |
176 | index c7ddb95..a7106f6 100644 |
177 | --- a/lib/lp/registry/interfaces/distribution.py |
178 | +++ b/lib/lp/registry/interfaces/distribution.py |
179 | @@ -1,4 +1,4 @@ |
180 | -# Copyright 2009-2015 Canonical Ltd. This software is licensed under the |
181 | +# Copyright 2009-2020 Canonical Ltd. This software is licensed under the |
182 | # GNU Affero General Public License version 3 (see the file LICENSE). |
183 | |
184 | """Interfaces including and related to IDistribution.""" |
185 | @@ -491,13 +491,13 @@ class IDistributionPublic( |
186 | """Return the country DNS mirror for a country and content type.""" |
187 | |
188 | def newMirror(owner, speed, country, content, display_name=None, |
189 | - description=None, http_base_url=None, |
190 | + description=None, http_base_url=None, https_base_url=None, |
191 | ftp_base_url=None, rsync_base_url=None, enabled=False, |
192 | official_candidate=False, whiteboard=None): |
193 | """Create a new DistributionMirror for this distribution. |
194 | |
195 | - At least one of http_base_url or ftp_base_url must be provided in |
196 | - order to create a mirror. |
197 | + At least one of {http,https,ftp}_base_url must be provided in order to |
198 | + create a mirror. |
199 | """ |
200 | |
201 | def getOCIProject(name): |
202 | diff --git a/lib/lp/registry/interfaces/distributionmirror.py b/lib/lp/registry/interfaces/distributionmirror.py |
203 | index ce6720e..4849e7a 100644 |
204 | --- a/lib/lp/registry/interfaces/distributionmirror.py |
205 | +++ b/lib/lp/registry/interfaces/distributionmirror.py |
206 | @@ -1,4 +1,4 @@ |
207 | -# Copyright 2009 Canonical Ltd. This software is licensed under the |
208 | +# Copyright 2009-2020 Canonical Ltd. This software is licensed under the |
209 | # GNU Affero General Public License version 3 (see the file LICENSE). |
210 | |
211 | __metaclass__ = type |
212 | @@ -302,6 +302,12 @@ class DistroMirrorHTTPURIField(DistroMirrorURIField): |
213 | return getUtility(IDistributionMirrorSet).getByHttpUrl(url) |
214 | |
215 | |
216 | +class DistroMirrorHTTPSURIField(DistroMirrorURIField): |
217 | + |
218 | + def getMirrorByURI(self, url): |
219 | + return getUtility(IDistributionMirrorSet).getByHttpsUrl(url) |
220 | + |
221 | + |
222 | class DistroMirrorFTPURIField(DistroMirrorURIField): |
223 | |
224 | def getMirrorByURI(self, url): |
225 | @@ -349,6 +355,14 @@ class IDistributionMirror(Interface): |
226 | allowed_schemes=['http'], allow_userinfo=False, |
227 | allow_query=False, allow_fragment=False, trailing_slash=True, |
228 | description=_('e.g.: http://archive.ubuntu.com/ubuntu/'))) |
229 | + https_base_url = exported(DistroMirrorHTTPSURIField( |
230 | + title=_('HTTPS URL'), required=False, readonly=False, |
231 | + allowed_schemes=['https'], allow_userinfo=False, |
232 | + allow_query=False, allow_fragment=False, trailing_slash=True, |
233 | + # XXX: pappacena 2020-02-21: Add description field with a more |
234 | + # suitable example once we have https for archive.ubuntu.com, like: |
235 | + # description=_('e.g.: http://archive.ubuntu.com/ubuntu/') |
236 | + )) |
237 | ftp_base_url = exported(DistroMirrorFTPURIField( |
238 | title=_('FTP URL'), required=False, readonly=False, |
239 | allowed_schemes=['ftp'], allow_userinfo=False, |
240 | @@ -435,8 +449,9 @@ class IDistributionMirror(Interface): |
241 | |
242 | @invariant |
243 | def mirrorMustHaveHTTPOrFTPURL(mirror): |
244 | - if not (mirror.http_base_url or mirror.ftp_base_url): |
245 | - raise Invalid('A mirror must have at least an HTTP or FTP URL.') |
246 | + if not (mirror.http_base_url or mirror.https_base_url or |
247 | + mirror.ftp_base_url): |
248 | + raise Invalid('A mirror must have at least an HTTP(S) or FTP URL.') |
249 | |
250 | def getSummarizedMirroredSourceSeries(): |
251 | """Return a summarized list of this distribution_mirror's |
252 | @@ -614,6 +629,9 @@ class IDistributionMirrorSet(Interface): |
253 | def getByHttpUrl(url): |
254 | """Return the mirror with the given HTTP URL or None.""" |
255 | |
256 | + def getByHttpsUrl(url): |
257 | + """Return the mirror with the given HTTPS URL or None.""" |
258 | + |
259 | def getByFtpUrl(url): |
260 | """Return the mirror with the given FTP URL or None.""" |
261 | |
262 | diff --git a/lib/lp/registry/model/distribution.py b/lib/lp/registry/model/distribution.py |
263 | index 2288d0e..f78908b 100644 |
264 | --- a/lib/lp/registry/model/distribution.py |
265 | +++ b/lib/lp/registry/model/distribution.py |
266 | @@ -1,4 +1,4 @@ |
267 | -# Copyright 2009-2016 Canonical Ltd. This software is licensed under the |
268 | +# Copyright 2009-2020 Canonical Ltd. This software is licensed under the |
269 | # GNU Affero General Public License version 3 (see the file LICENSE). |
270 | |
271 | """Database classes for implementing distribution items.""" |
272 | @@ -709,7 +709,7 @@ class Distribution(SQLBase, BugTargetBase, MakesAnnouncements, |
273 | country_dns_mirror=True).one() |
274 | |
275 | def newMirror(self, owner, speed, country, content, display_name=None, |
276 | - description=None, http_base_url=None, |
277 | + description=None, http_base_url=None, https_base_url=None, |
278 | ftp_base_url=None, rsync_base_url=None, |
279 | official_candidate=False, enabled=False, |
280 | whiteboard=None): |
281 | @@ -722,15 +722,17 @@ class Distribution(SQLBase, BugTargetBase, MakesAnnouncements, |
282 | return None |
283 | |
284 | urls = {'http_base_url': http_base_url, |
285 | + 'https_base_url': https_base_url, |
286 | 'ftp_base_url': ftp_base_url, |
287 | 'rsync_base_url': rsync_base_url} |
288 | for name, value in urls.items(): |
289 | if value is not None: |
290 | urls[name] = IDistributionMirror[name].normalize(value) |
291 | |
292 | - url = urls['http_base_url'] or urls['ftp_base_url'] |
293 | + url = (urls['https_base_url'] or urls['http_base_url'] or |
294 | + urls['ftp_base_url']) |
295 | assert url is not None, ( |
296 | - "A mirror must provide either an HTTP or FTP URL (or both).") |
297 | + "A mirror must provide at least one HTTP/HTTPS/FTP URL.") |
298 | dummy, host, dummy, dummy, dummy, dummy = urlparse(url) |
299 | name = sanitize_name('%s-%s' % (host, content.name.lower())) |
300 | |
301 | @@ -744,6 +746,7 @@ class Distribution(SQLBase, BugTargetBase, MakesAnnouncements, |
302 | distribution=self, owner=owner, name=name, speed=speed, |
303 | country=country, content=content, display_name=display_name, |
304 | description=description, http_base_url=urls['http_base_url'], |
305 | + https_base_url=urls['https_base_url'], |
306 | ftp_base_url=urls['ftp_base_url'], |
307 | rsync_base_url=urls['rsync_base_url'], |
308 | official_candidate=official_candidate, enabled=enabled, |
309 | diff --git a/lib/lp/registry/model/distributionmirror.py b/lib/lp/registry/model/distributionmirror.py |
310 | index 747a2b0..323fb26 100644 |
311 | --- a/lib/lp/registry/model/distributionmirror.py |
312 | +++ b/lib/lp/registry/model/distributionmirror.py |
313 | @@ -1,4 +1,4 @@ |
314 | -# Copyright 2009-2015 Canonical Ltd. This software is licensed under the |
315 | +# Copyright 2009-2020 Canonical Ltd. This software is licensed under the |
316 | # GNU Affero General Public License version 3 (see the file LICENSE). |
317 | |
318 | """Module docstring goes here.""" |
319 | @@ -129,6 +129,8 @@ class DistributionMirror(SQLBase): |
320 | notNull=False, default=None) |
321 | http_base_url = StringCol( |
322 | notNull=False, default=None, unique=True) |
323 | + https_base_url = StringCol( |
324 | + notNull=False, default=None, unique=True) |
325 | ftp_base_url = StringCol( |
326 | notNull=False, default=None, unique=True) |
327 | rsync_base_url = StringCol( |
328 | @@ -155,7 +157,9 @@ class DistributionMirror(SQLBase): |
329 | @property |
330 | def base_url(self): |
331 | """See IDistributionMirror""" |
332 | - if self.http_base_url is not None: |
333 | + if self.https_base_url is not None: |
334 | + return self.https_base_url |
335 | + elif self.http_base_url is not None: |
336 | return self.http_base_url |
337 | else: |
338 | return self.ftp_base_url |
339 | @@ -677,6 +681,10 @@ class DistributionMirrorSet: |
340 | """See IDistributionMirrorSet""" |
341 | return DistributionMirror.selectOneBy(http_base_url=url) |
342 | |
343 | + def getByHttpsUrl(self, url): |
344 | + """See IDistributionMirrorSet""" |
345 | + return DistributionMirror.selectOneBy(https_base_url=url) |
346 | + |
347 | def getByFtpUrl(self, url): |
348 | """See IDistributionMirrorSet""" |
349 | return DistributionMirror.selectOneBy(ftp_base_url=url) |
350 | diff --git a/lib/lp/registry/scripts/distributionmirror_prober.py b/lib/lp/registry/scripts/distributionmirror_prober.py |
351 | index 916d4ba..c633fb8 100644 |
352 | --- a/lib/lp/registry/scripts/distributionmirror_prober.py |
353 | +++ b/lib/lp/registry/scripts/distributionmirror_prober.py |
354 | @@ -1,4 +1,4 @@ |
355 | -# Copyright 2009-2019 Canonical Ltd. This software is licensed under the |
356 | +# Copyright 2009-2020 Canonical Ltd. This software is licensed under the |
357 | # GNU Affero General Public License version 3 (see the file LICENSE). |
358 | |
359 | __metaclass__ = type |
360 | @@ -12,6 +12,7 @@ import logging |
361 | import os.path |
362 | from StringIO import StringIO |
363 | |
364 | +import OpenSSL |
365 | import requests |
366 | from six.moves import http_client |
367 | from six.moves.urllib.parse import ( |
368 | @@ -20,14 +21,27 @@ from six.moves.urllib.parse import ( |
369 | urlparse, |
370 | urlunparse, |
371 | ) |
372 | +from treq.client import HTTPClient as TreqHTTPClient |
373 | from twisted.internet import ( |
374 | defer, |
375 | protocol, |
376 | reactor, |
377 | ) |
378 | -from twisted.internet.defer import DeferredSemaphore |
379 | +from twisted.internet.defer import ( |
380 | + CancelledError, |
381 | + DeferredSemaphore, |
382 | + ) |
383 | +from twisted.internet.endpoints import HostnameEndpoint |
384 | +from twisted.internet.ssl import VerificationError |
385 | from twisted.python.failure import Failure |
386 | +from twisted.web.client import ( |
387 | + Agent, |
388 | + BrowserLikePolicyForHTTPS, |
389 | + ProxyAgent, |
390 | + ResponseNeverReceived, |
391 | + ) |
392 | from twisted.web.http import HTTPClient |
393 | +from twisted.web.iweb import IResponse |
394 | from zope.component import getUtility |
395 | |
396 | from lp.app.interfaces.launchpad import ILaunchpadCelebrities |
397 | @@ -50,6 +64,7 @@ from lp.soyuz.interfaces.distroarchseries import IDistroArchSeries |
398 | # IMPORTANT: Changing these values can cause lots of false negatives when |
399 | # probing mirrors, so please don't change them unless you know what you're |
400 | # doing. |
401 | + |
402 | MIN_REQUEST_TIMEOUT_RATIO = 3 |
403 | MIN_REQUESTS_TO_CONSIDER_RATIO = 30 |
404 | |
405 | @@ -57,6 +72,9 @@ MIN_REQUESTS_TO_CONSIDER_RATIO = 30 |
406 | # We need to get rid of these global dicts in this module. |
407 | host_requests = {} |
408 | host_timeouts = {} |
409 | +# Set of invalid certificate (host, port) tuples, to avoid doing HTTPS calls |
410 | +# to hosts we already know they are not valid. |
411 | +invalid_certificate_hosts = set() |
412 | |
413 | MAX_REDIRECTS = 3 |
414 | |
415 | @@ -161,6 +179,64 @@ class ProberProtocol(HTTPClient): |
416 | pass |
417 | |
418 | |
419 | +class HTTPSProbeFailureHandler: |
420 | + """Handler to translate general errors into expected errors on HTTPS |
421 | + connections.""" |
422 | + def __init__(self, factory): |
423 | + self.factory = factory |
424 | + |
425 | + def handleResponse(self, response): |
426 | + """Translates any request with return code different from 200 into |
427 | + an error in the callback chain. |
428 | + |
429 | + Note that other 2xx codes that are not 200 are considered errors too. |
430 | + This behaviour is the same as seen in ProberProtocol.handleStatus, |
431 | + for HTTP responses. |
432 | + """ |
433 | + status = response.code |
434 | + if status == http_client.OK: |
435 | + return response |
436 | + else: |
437 | + raise BadResponseCode(status, response) |
438 | + |
439 | + def handleErrors(self, error): |
440 | + """Handle exceptions in https requests. |
441 | + """ |
442 | + if self.isInvalidCertificateError(error): |
443 | + invalid_certificate_hosts.add( |
444 | + (self.factory.request_host, self.factory.request_port)) |
445 | + reason = InvalidHTTPSCertificate( |
446 | + self.factory.request_host, self.factory.request_port) |
447 | + raise reason |
448 | + if self.isTimeout(error): |
449 | + raise ProberTimeout(self.factory.url, self.factory.timeout) |
450 | + raise error |
451 | + |
452 | + def isTimeout(self, error): |
453 | + """Checks if the error was caused by a timeout. |
454 | + """ |
455 | + return self._isErrorFromType(error, CancelledError) |
456 | + |
457 | + def isInvalidCertificateError(self, error): |
458 | + """Checks if the error was caused by an invalid certificate. |
459 | + """ |
460 | + # It might be a raw SSL error, or a twisted-encapsulated |
461 | + # verification error (such as DNSMismatch error when the |
462 | + # certificate is valid for a different domain, for example). |
463 | + return self._isErrorFromType( |
464 | + error, OpenSSL.SSL.Error, VerificationError) |
465 | + |
466 | + def _isErrorFromType(self, error, *types): |
467 | + """Checks if the error was caused by any of the given types. |
468 | + """ |
469 | + if not isinstance(error.value, ResponseNeverReceived): |
470 | + return False |
471 | + for reason in error.value.reasons: |
472 | + if reason.check(*types) is not None: |
473 | + return True |
474 | + return False |
475 | + |
476 | + |
477 | class RedirectAwareProberProtocol(ProberProtocol): |
478 | """A specialized version of ProberProtocol that follows HTTP redirects.""" |
479 | |
480 | @@ -216,6 +292,8 @@ class ProberFactory(protocol.ClientFactory): |
481 | connect_port = None |
482 | connect_path = None |
483 | |
484 | + https_agent_policy = BrowserLikePolicyForHTTPS |
485 | + |
486 | def __init__(self, url, timeout=config.distributionmirrorprober.timeout): |
487 | # We want the deferred to be a private attribute (_deferred) to make |
488 | # sure our clients will only use the deferred returned by the probe() |
489 | @@ -225,9 +303,14 @@ class ProberFactory(protocol.ClientFactory): |
490 | self.timeout = timeout |
491 | self.timeoutCall = None |
492 | self.setURL(url.encode('ascii')) |
493 | + self.logger = logging.getLogger('distributionmirror-prober') |
494 | + |
495 | + @property |
496 | + def is_https(self): |
497 | + return self.request_scheme == 'https' |
498 | |
499 | def probe(self): |
500 | - logger = logging.getLogger('distributionmirror-prober') |
501 | + logger = self.logger |
502 | # NOTE: We don't want to issue connections to any outside host when |
503 | # running the mirror prober in a development machine, so we do this |
504 | # hack here. |
505 | @@ -244,13 +327,46 @@ class ProberFactory(protocol.ClientFactory): |
506 | "host already." % self.url) |
507 | return self._deferred |
508 | |
509 | + if (self.request_host, self.request_port) in invalid_certificate_hosts: |
510 | + reactor.callLater( |
511 | + 0, self.failed, InvalidHTTPSCertificateSkipped(self.url)) |
512 | + logger.debug("Skipping %s as it doesn't have a valid HTTPS " |
513 | + "certificate" % self.url) |
514 | + return self._deferred |
515 | + |
516 | self.connect() |
517 | logger.debug('Probing %s' % self.url) |
518 | return self._deferred |
519 | |
520 | + def getHttpsClient(self): |
521 | + # Should we use a proxy? |
522 | + if not config.launchpad.http_proxy: |
523 | + agent = Agent( |
524 | + reactor=reactor, contextFactory=self.https_agent_policy()) |
525 | + else: |
526 | + endpoint = HostnameEndpoint( |
527 | + reactor, self.connect_host, self.connect_port) |
528 | + agent = ProxyAgent(endpoint) |
529 | + return TreqHTTPClient(agent) |
530 | + |
531 | def connect(self): |
532 | + """Starts the connection and sets the self._deferred to the proper |
533 | + task. |
534 | + """ |
535 | host_requests[self.request_host] += 1 |
536 | - reactor.connectTCP(self.connect_host, self.connect_port, self) |
537 | + if self.is_https: |
538 | + treq = self.getHttpsClient() |
539 | + self._deferred.addCallback( |
540 | + lambda _: treq.head( |
541 | + self.url, reactor=reactor, allow_redirects=True, |
542 | + timeout=self.timeout)) |
543 | + error_handler = HTTPSProbeFailureHandler(self) |
544 | + self._deferred.addCallback(error_handler.handleResponse) |
545 | + self._deferred.addErrback(error_handler.handleErrors) |
546 | + reactor.callWhenRunning(self._deferred.callback, None) |
547 | + else: |
548 | + reactor.connectTCP(self.connect_host, self.connect_port, self) |
549 | + |
550 | if self.timeoutCall is not None and self.timeoutCall.active(): |
551 | self._cancelTimeout(None) |
552 | self.timeoutCall = reactor.callLater( |
553 | @@ -269,6 +385,8 @@ class ProberFactory(protocol.ClientFactory): |
554 | self.connector = connector |
555 | |
556 | def succeeded(self, status): |
557 | + if IResponse.providedBy(status): |
558 | + status = str(status.code) |
559 | self._deferred.callback(status) |
560 | |
561 | def failed(self, reason): |
562 | @@ -288,7 +406,7 @@ class ProberFactory(protocol.ClientFactory): |
563 | # https://bugs.squid-cache.org/show_bug.cgi?id=1758 applied. So, if |
564 | # you encounter any problems with FTP URLs you'll probably have to nag |
565 | # the sysadmins to fix squid for you. |
566 | - if scheme not in ('http', 'ftp'): |
567 | + if scheme not in ('http', 'https', 'ftp'): |
568 | raise UnknownURLScheme(url) |
569 | |
570 | if scheme and host: |
571 | @@ -376,14 +494,26 @@ class ProberTimeout(ProberError): |
572 | |
573 | class BadResponseCode(ProberError): |
574 | |
575 | - def __init__(self, status, *args): |
576 | + def __init__(self, status, response=None, *args): |
577 | ProberError.__init__(self, *args) |
578 | self.status = status |
579 | + self.response = response |
580 | |
581 | def __str__(self): |
582 | return "Bad response code: %s" % self.status |
583 | |
584 | |
585 | +class InvalidHTTPSCertificate(ProberError): |
586 | + def __init__(self, host, port, *args): |
587 | + super(InvalidHTTPSCertificate, self).__init__(*args) |
588 | + self.host = host |
589 | + self.port = port |
590 | + |
591 | + def __str__(self): |
592 | + return "Invalid SSL certificate when trying to probe %s:%s" % ( |
593 | + self.host, self.port) |
594 | + |
595 | + |
596 | class RedirectToDifferentFile(ProberError): |
597 | |
598 | def __init__(self, orig_path, new_path, *args): |
599 | @@ -409,6 +539,14 @@ class ConnectionSkipped(ProberError): |
600 | "host. It will be retried on the next probing run.") |
601 | |
602 | |
603 | +class InvalidHTTPSCertificateSkipped(ProberError): |
604 | + |
605 | + def __str__(self): |
606 | + return ("Connection skipped because the server doesn't have a valid " |
607 | + "HTTPS certificate. It will be retried on the next " |
608 | + "probing run.") |
609 | + |
610 | + |
611 | class UnknownURLScheme(ProberError): |
612 | |
613 | def __init__(self, url, *args): |
614 | @@ -429,7 +567,13 @@ class UnknownURLSchemeAfterRedirect(UnknownURLScheme): |
615 | |
616 | class ArchiveMirrorProberCallbacks(LoggingMixin): |
617 | |
618 | - expected_failures = (BadResponseCode, ProberTimeout, ConnectionSkipped) |
619 | + expected_failures = ( |
620 | + BadResponseCode, |
621 | + ProberTimeout, |
622 | + ConnectionSkipped, |
623 | + InvalidHTTPSCertificate, |
624 | + InvalidHTTPSCertificateSkipped, |
625 | + ) |
626 | |
627 | def __init__(self, mirror, series, pocket, component, url, log_file): |
628 | self.mirror = mirror |
629 | @@ -574,6 +718,8 @@ class MirrorCDImageProberCallbacks(LoggingMixin): |
630 | ProberTimeout, |
631 | RedirectToDifferentFile, |
632 | UnknownURLSchemeAfterRedirect, |
633 | + InvalidHTTPSCertificate, |
634 | + InvalidHTTPSCertificateSkipped, |
635 | ) |
636 | |
637 | def __init__(self, mirror, distroseries, flavour, log_file): |
638 | diff --git a/lib/lp/registry/stories/webservice/xx-distribution-mirror.txt b/lib/lp/registry/stories/webservice/xx-distribution-mirror.txt |
639 | index db17b40..3e49ebe 100644 |
640 | --- a/lib/lp/registry/stories/webservice/xx-distribution-mirror.txt |
641 | +++ b/lib/lp/registry/stories/webservice/xx-distribution-mirror.txt |
642 | @@ -23,6 +23,7 @@ mirrors: |
643 | enabled: True |
644 | ftp_base_url: None |
645 | http_base_url: u'http://archive.ubuntu.com/ubuntu/' |
646 | + https_base_url: None |
647 | name: u'canonical-archive' |
648 | official_candidate: True |
649 | owner_link: u'http://.../~mark' |
650 | @@ -52,6 +53,7 @@ And CD image mirrors: |
651 | enabled: True |
652 | ftp_base_url: None |
653 | http_base_url: u'http://releases.ubuntu.com/' |
654 | + https_base_url: None |
655 | name: u'canonical-releases' |
656 | official_candidate: True |
657 | owner_link: u'http://.../~mark' |
658 | @@ -145,6 +147,7 @@ Mirror listing admins may see all: |
659 | enabled: True |
660 | ftp_base_url: None |
661 | http_base_url: u'http://localhost:11375/archive-mirror/' |
662 | + https_base_url: None |
663 | name: u'archive-404-mirror' |
664 | official_candidate: True |
665 | owner_link: u'http://.../~name12' |
666 | @@ -227,6 +230,7 @@ While others can be set with the appropriate authorization: |
667 | enabled: True |
668 | ftp_base_url: None |
669 | http_base_url: u'http://releases.ubuntu.com/' |
670 | + https_base_url: None |
671 | name: u'canonical-releases' |
672 | official_candidate: True |
673 | owner_link: u'http://.../~mark' |
674 | diff --git a/lib/lp/registry/stories/webservice/xx-distribution.txt b/lib/lp/registry/stories/webservice/xx-distribution.txt |
675 | index 1c0ed87..1486904 100644 |
676 | --- a/lib/lp/registry/stories/webservice/xx-distribution.txt |
677 | +++ b/lib/lp/registry/stories/webservice/xx-distribution.txt |
678 | @@ -159,6 +159,7 @@ packages matching (substring) the given text. |
679 | enabled: True |
680 | ftp_base_url: None |
681 | http_base_url: u'http://releases.ubuntu.com/' |
682 | + https_base_url: None |
683 | name: u'canonical-releases' |
684 | official_candidate: True |
685 | owner_link: u'http://.../~mark' |
686 | diff --git a/lib/lp/registry/templates/distributionmirror-index.pt b/lib/lp/registry/templates/distributionmirror-index.pt |
687 | index 78e934d..27f3139 100644 |
688 | --- a/lib/lp/registry/templates/distributionmirror-index.pt |
689 | +++ b/lib/lp/registry/templates/distributionmirror-index.pt |
690 | @@ -118,6 +118,10 @@ |
691 | <h2>Mirror location information</h2> |
692 | |
693 | <ul class="webref" id="mirror-urls"> |
694 | + <li tal:condition="context/https_base_url" > |
695 | + <a tal:content="context/https_base_url" |
696 | + tal:attributes="href context/https_base_url">https://url/</a> |
697 | + </li> |
698 | <li tal:condition="context/http_base_url" > |
699 | <a tal:content="context/http_base_url" |
700 | tal:attributes="href context/http_base_url">http://url/</a> |
701 | diff --git a/lib/lp/registry/templates/distributionmirror-macros.pt b/lib/lp/registry/templates/distributionmirror-macros.pt |
702 | index 18f82af..489f770 100644 |
703 | --- a/lib/lp/registry/templates/distributionmirror-macros.pt |
704 | +++ b/lib/lp/registry/templates/distributionmirror-macros.pt |
705 | @@ -17,7 +17,7 @@ |
706 | <tbody> |
707 | <tal:country_and_mirrors repeat="country_and_mirrors mirrors_by_country"> |
708 | <tr class="head"> |
709 | - <th colspan="2" |
710 | + <th colspan="2" |
711 | tal:content="country_and_mirrors/country" /> |
712 | <th tal:content="country_and_mirrors/throughput"/> |
713 | <th tal:condition="show_mirror_type"> |
714 | @@ -35,6 +35,8 @@ |
715 | tal:content="mirror/title">Mirror Name</a> |
716 | </td> |
717 | <td> |
718 | + <a tal:condition="mirror/https_base_url" |
719 | + tal:attributes="href mirror/https_base_url">https</a> |
720 | <a tal:condition="mirror/http_base_url" |
721 | tal:attributes="href mirror/http_base_url">http</a> |
722 | <a tal:condition="mirror/ftp_base_url" |
723 | diff --git a/lib/lp/registry/tests/distributionmirror_http_server.py b/lib/lp/registry/tests/distributionmirror_http_server.py |
724 | index e3e0a39..d12c9b3 100644 |
725 | --- a/lib/lp/registry/tests/distributionmirror_http_server.py |
726 | +++ b/lib/lp/registry/tests/distributionmirror_http_server.py |
727 | @@ -1,6 +1,6 @@ |
728 | #!/usr/bin/python |
729 | # |
730 | -# Copyright 2009 Canonical Ltd. This software is licensed under the |
731 | +# Copyright 2009-2020 Canonical Ltd. This software is licensed under the |
732 | # GNU Affero General Public License version 3 (see the file LICENSE). |
733 | |
734 | from twisted.web.resource import Resource |
735 | @@ -21,10 +21,10 @@ class DistributionMirrorTestHTTPServer(Resource): |
736 | :error: Respond with a '500 Internal Server Error' status. |
737 | |
738 | :redirect-to-valid-mirror/*: Respond with a '302 Found' status, |
739 | - redirecting to http://localhost:%(port)s/valid-mirror/*. |
740 | + redirecting to http(s)://localhost:%(port)s/valid-mirror/*. |
741 | |
742 | :redirect-infinite-loop: Respond with a '302 Found' status, redirecting |
743 | - to http://localhost:%(port)s/redirect-infinite-loop. |
744 | + to http(s)://localhost:%(port)s/redirect-infinite-loop. |
745 | |
746 | :redirect-unknown-url-scheme: Respond with a '302 Found' status, |
747 | redirecting to ssh://localhost/redirect-unknown-url-scheme. |
748 | @@ -32,11 +32,13 @@ class DistributionMirrorTestHTTPServer(Resource): |
749 | Any other path will cause the server to respond with a '404 Not Found' |
750 | status. |
751 | """ |
752 | + protocol = "http" |
753 | |
754 | def getChild(self, name, request): |
755 | + protocol = self.protocol |
756 | port = request.getHost().port |
757 | if name == 'valid-mirror': |
758 | - leaf = DistributionMirrorTestHTTPServer() |
759 | + leaf = self.__class__() |
760 | leaf.isLeaf = True |
761 | return leaf |
762 | elif name == 'timeout': |
763 | @@ -49,12 +51,14 @@ class DistributionMirrorTestHTTPServer(Resource): |
764 | 'than one component.') |
765 | remaining_path = request.path.replace('/%s' % name, '') |
766 | leaf = RedirectingResource( |
767 | - 'http://localhost:%s/valid-mirror%s' % (port, remaining_path)) |
768 | + '%s://localhost:%s/valid-mirror%s' % ( |
769 | + protocol, port, remaining_path)) |
770 | leaf.isLeaf = True |
771 | return leaf |
772 | elif name == 'redirect-infinite-loop': |
773 | return RedirectingResource( |
774 | - 'http://localhost:%s/redirect-infinite-loop' % port) |
775 | + '%s://localhost:%s/redirect-infinite-loop' % |
776 | + (protocol, port)) |
777 | elif name == 'redirect-unknown-url-scheme': |
778 | return RedirectingResource( |
779 | 'ssh://localhost/redirect-unknown-url-scheme') |
780 | @@ -65,6 +69,11 @@ class DistributionMirrorTestHTTPServer(Resource): |
781 | return "Hi" |
782 | |
783 | |
784 | +class DistributionMirrorTestSecureHTTPServer(DistributionMirrorTestHTTPServer): |
785 | + """HTTPS version of DistributionMirrorTestHTTPServer""" |
786 | + protocol = "https" |
787 | + |
788 | + |
789 | class RedirectingResource(Resource): |
790 | |
791 | def __init__(self, redirection_url): |
792 | @@ -85,4 +94,3 @@ class FiveHundredResource(Resource): |
793 | def render_GET(self, request): |
794 | request.setResponseCode(500) |
795 | request.write('ASPLODE!!!') |
796 | - return NOT_DONE_YET |
797 | diff --git a/lib/lp/registry/tests/test_distributionmirror_prober.py b/lib/lp/registry/tests/test_distributionmirror_prober.py |
798 | index 39228d2..c3e363c 100644 |
799 | --- a/lib/lp/registry/tests/test_distributionmirror_prober.py |
800 | +++ b/lib/lp/registry/tests/test_distributionmirror_prober.py |
801 | @@ -1,4 +1,4 @@ |
802 | -# Copyright 2009-2018 Canonical Ltd. This software is licensed under the |
803 | +# Copyright 2009-2020 Canonical Ltd. This software is licensed under the |
804 | # GNU Affero General Public License version 3 (see the file LICENSE). |
805 | |
806 | """distributionmirror-prober tests.""" |
807 | @@ -11,6 +11,7 @@ import logging |
808 | import os |
809 | from StringIO import StringIO |
810 | |
811 | +from fixtures import MockPatchObject |
812 | from lazr.uri import URI |
813 | import responses |
814 | from six.moves import http_client |
815 | @@ -28,9 +29,14 @@ from testtools.twistedsupport import ( |
816 | from twisted.internet import ( |
817 | defer, |
818 | reactor, |
819 | + ssl, |
820 | ) |
821 | from twisted.python.failure import Failure |
822 | from twisted.web import server |
823 | +from twisted.web.client import ( |
824 | + BrowserLikePolicyForHTTPS, |
825 | + ProxyAgent, |
826 | + ) |
827 | from zope.component import getUtility |
828 | from zope.security.proxy import removeSecurityProxy |
829 | |
830 | @@ -44,6 +50,8 @@ from lp.registry.scripts.distributionmirror_prober import ( |
831 | BadResponseCode, |
832 | ConnectionSkipped, |
833 | InfiniteLoopDetected, |
834 | + InvalidHTTPSCertificate, |
835 | + InvalidHTTPSCertificateSkipped, |
836 | LoggingMixin, |
837 | MAX_REDIRECTS, |
838 | MIN_REQUEST_TIMEOUT_RATIO, |
839 | @@ -65,6 +73,7 @@ from lp.registry.scripts.distributionmirror_prober import ( |
840 | ) |
841 | from lp.registry.tests.distributionmirror_http_server import ( |
842 | DistributionMirrorTestHTTPServer, |
843 | + DistributionMirrorTestSecureHTTPServer, |
844 | ) |
845 | from lp.services.config import config |
846 | from lp.services.daemons.tachandler import TacTestSetup |
847 | @@ -103,6 +112,186 @@ class HTTPServerTestSetup(TacTestSetup): |
848 | return os.path.join(self.root, 'distributionmirror_http_server.log') |
849 | |
850 | |
851 | + |
852 | +class LocalhostWhitelistedHTTPSPolicy(BrowserLikePolicyForHTTPS): |
853 | + """HTTPS policy that bypasses SSL certificate check when doing requests |
854 | + to localhost. |
855 | + """ |
856 | + |
857 | + def creatorForNetloc(self, hostname, port): |
858 | + # check if the hostname is in the the whitelist, |
859 | + # otherwise return the default policy |
860 | + if hostname == 'localhost': |
861 | + return ssl.CertificateOptions(verify=False) |
862 | + return super(LocalhostWhitelistedHTTPSPolicy, self).creatorForNetloc( |
863 | + hostname, port) |
864 | + |
865 | + |
866 | +class TestProberHTTPSProtocolAndFactory(TestCase): |
867 | + layer = TwistedLayer |
868 | + run_tests_with = AsynchronousDeferredRunTestForBrokenTwisted.make_factory( |
869 | + timeout=30) |
870 | + |
871 | + def setUp(self): |
872 | + super(TestProberHTTPSProtocolAndFactory, self).setUp() |
873 | + root = DistributionMirrorTestSecureHTTPServer() |
874 | + site = server.Site(root) |
875 | + site.displayTracebacks = False |
876 | + keys_path = os.path.join(config.root, "configs", "development") |
877 | + keys = ssl.DefaultOpenSSLContextFactory( |
878 | + os.path.join(keys_path, "launchpad.key"), |
879 | + os.path.join(keys_path, "launchpad.crt"), |
880 | + ) |
881 | + self.listening_port = reactor.listenSSL(0, site, keys) |
882 | + |
883 | + self.addCleanup(self.listening_port.stopListening) |
884 | + |
885 | + # Change the default policy to accept localhost self-signed |
886 | + # certificates. |
887 | + original_probefactory_policy = ProberFactory.https_agent_policy |
888 | + original_redirect_policy = ( |
889 | + RedirectAwareProberFactory.https_agent_policy) |
890 | + ProberFactory.https_agent_policy = LocalhostWhitelistedHTTPSPolicy |
891 | + RedirectAwareProberFactory.https_agent_policy = ( |
892 | + LocalhostWhitelistedHTTPSPolicy) |
893 | + |
894 | + for factory in (ProberFactory, RedirectAwareProberFactory): |
895 | + self.useFixture(MockPatchObject( |
896 | + factory, "https_agent_policy", |
897 | + LocalhostWhitelistedHTTPSPolicy)) |
898 | + |
899 | + self.port = self.listening_port.getHost().port |
900 | + |
901 | + self.urls = {'timeout': u'https://localhost:%s/timeout' % self.port, |
902 | + '200': u'https://localhost:%s/valid-mirror' % self.port, |
903 | + '500': u'https://localhost:%s/error' % self.port, |
904 | + '404': u'https://localhost:%s/invalid-mirror' % self.port} |
905 | + self.pushConfig('launchpad', http_proxy=None) |
906 | + |
907 | + self.useFixture(MockPatchObject( |
908 | + distributionmirror_prober, "host_requests", {})) |
909 | + self.useFixture(MockPatchObject( |
910 | + distributionmirror_prober, "host_timeouts", {})) |
911 | + self.useFixture(MockPatchObject( |
912 | + distributionmirror_prober, "invalid_certificate_hosts", set())) |
913 | + |
914 | + def _createProberAndProbe(self, url): |
915 | + prober = ProberFactory(url) |
916 | + return prober.probe() |
917 | + |
918 | + def test_timeout(self): |
919 | + prober = ProberFactory(self.urls['timeout'], timeout=0.5) |
920 | + d = prober.probe() |
921 | + return assert_fails_with(d, ProberTimeout) |
922 | + |
923 | + def test_500(self): |
924 | + d = self._createProberAndProbe(self.urls['500']) |
925 | + return assert_fails_with(d, BadResponseCode) |
926 | + |
927 | + def test_notfound(self): |
928 | + d = self._createProberAndProbe(self.urls['404']) |
929 | + return assert_fails_with(d, BadResponseCode) |
930 | + |
931 | + def test_config_no_https_proxy(self): |
932 | + prober = ProberFactory(self.urls['200']) |
933 | + self.assertThat(prober, MatchesStructure.byEquality( |
934 | + request_scheme='https', |
935 | + request_host='localhost', |
936 | + request_port=self.port, |
937 | + request_path='/valid-mirror', |
938 | + connect_scheme='https', |
939 | + connect_host='localhost', |
940 | + connect_port=self.port, |
941 | + connect_path='/valid-mirror')) |
942 | + |
943 | + def test_RedirectAwareProber_follows_https_redirect(self): |
944 | + url = 'https://localhost:%s/redirect-to-valid-mirror/file' % self.port |
945 | + prober = RedirectAwareProberFactory(url) |
946 | + self.assertEqual(prober.url, url) |
947 | + deferred = prober.probe() |
948 | + |
949 | + def got_result(result): |
950 | + self.assertEqual(http_client.OK, result.code) |
951 | + self.assertEqual( |
952 | + 'https://localhost:%s/valid-mirror/file' % self.port, |
953 | + result.request.absoluteURI) |
954 | + |
955 | + return deferred.addCallback(got_result) |
956 | + |
957 | + def test_https_prober_uses_proxy(self): |
958 | + root = DistributionMirrorTestSecureHTTPServer() |
959 | + site = server.Site(root) |
960 | + proxy_listen_port = reactor.listenTCP(0, site) |
961 | + proxy_port = proxy_listen_port.getHost().port |
962 | + self.pushConfig( |
963 | + 'launchpad', http_proxy='http://localhost:%s/valid-mirror/file' |
964 | + % proxy_port) |
965 | + |
966 | + url = 'https://localhost:%s/valid-mirror/file' % self.port |
967 | + prober = RedirectAwareProberFactory(url) |
968 | + self.assertEqual(prober.url, url) |
969 | + deferred = prober.probe() |
970 | + |
971 | + def got_result(result): |
972 | + # We basically don't care about the result here. We just want to |
973 | + # check that it did the request to the correct URI, |
974 | + # and ProxyAgent was used pointing to the correct proxy. |
975 | + agent = prober.getHttpsClient()._agent |
976 | + self.assertIsInstance(agent, ProxyAgent) |
977 | + self.assertEqual('localhost', agent._proxyEndpoint._hostText) |
978 | + self.assertEqual(proxy_port, agent._proxyEndpoint._port) |
979 | + |
980 | + self.assertEqual( |
981 | + 'https://localhost:%s/valid-mirror/file' % self.port, |
982 | + result.value.response.request.absoluteURI) |
983 | + |
984 | + def cleanup(*args, **kwargs): |
985 | + proxy_listen_port.stopListening() |
986 | + |
987 | + # Doing the proxy checks on the error callback because the |
988 | + # proxy is dummy and always returns 404. |
989 | + deferred.addErrback(got_result) |
990 | + deferred.addBoth(cleanup) |
991 | + return deferred |
992 | + |
993 | + def test_https_fails_on_invalid_certificates(self): |
994 | + """Changes set back the default browser-like policy for HTTPS |
995 | + request and make sure the request is failing due to invalid |
996 | + (self-signed) certificate. |
997 | + """ |
998 | + url = 'https://localhost:%s/valid-mirror/file' % self.port |
999 | + prober = RedirectAwareProberFactory(url) |
1000 | + prober.https_agent_policy = BrowserLikePolicyForHTTPS |
1001 | + self.assertEqual(prober.url, url) |
1002 | + deferred = prober.probe() |
1003 | + |
1004 | + def on_failure(result): |
1005 | + self.assertIsInstance(result.value, InvalidHTTPSCertificate) |
1006 | + self.assertIn( |
1007 | + ("localhost", self.port), |
1008 | + distributionmirror_prober.invalid_certificate_hosts) |
1009 | + |
1010 | + def on_success(result): |
1011 | + if result is not None: |
1012 | + self.fail( |
1013 | + "Should have raised SSL error. Got '%s' instead" % result) |
1014 | + |
1015 | + deferred.addErrback(on_failure) |
1016 | + deferred.addCallback(on_success) |
1017 | + return deferred |
1018 | + |
1019 | + def test_https_skips_invalid_certificates_hosts(self): |
1020 | + distributionmirror_prober.invalid_certificate_hosts.add( |
1021 | + ("localhost", self.port)) |
1022 | + url = 'https://localhost:%s/valid-mirror/file' % self.port |
1023 | + prober = RedirectAwareProberFactory(url) |
1024 | + prober.https_agent_policy = BrowserLikePolicyForHTTPS |
1025 | + self.assertEqual(prober.url, url) |
1026 | + deferred = prober.probe() |
1027 | + |
1028 | + return assert_fails_with(deferred, InvalidHTTPSCertificateSkipped) |
1029 | + |
1030 | + |
1031 | class TestProberProtocolAndFactory(TestCase): |
1032 | |
1033 | layer = TwistedLayer |
1034 | @@ -212,7 +401,7 @@ class TestProberProtocolAndFactory(TestCase): |
1035 | self.assertTrue(prober.url == new_url) |
1036 | self.assertTrue(result == str(http_client.OK)) |
1037 | |
1038 | - return deferred.addCallback(got_result) |
1039 | + return deferred.addBoth(got_result) |
1040 | |
1041 | def test_redirectawareprober_detects_infinite_loop(self): |
1042 | prober = RedirectAwareProberFactory( |
1043 | @@ -737,12 +926,16 @@ class TestMirrorCDImageProberCallbacks(TestCaseWithFactory): |
1044 | ConnectionSkipped, |
1045 | RedirectToDifferentFile, |
1046 | UnknownURLSchemeAfterRedirect, |
1047 | + InvalidHTTPSCertificate, |
1048 | + InvalidHTTPSCertificateSkipped, |
1049 | ])) |
1050 | exceptions = [BadResponseCode(str(http_client.NOT_FOUND)), |
1051 | ProberTimeout('http://localhost/', 5), |
1052 | ConnectionSkipped(), |
1053 | RedirectToDifferentFile('/foo', '/bar'), |
1054 | - UnknownURLSchemeAfterRedirect('https://localhost')] |
1055 | + UnknownURLSchemeAfterRedirect('https://localhost'), |
1056 | + InvalidHTTPSCertificate('localhost', 443), |
1057 | + InvalidHTTPSCertificateSkipped("https://localhost/xx")] |
1058 | for exception in exceptions: |
1059 | failure = callbacks.ensureOrDeleteMirrorCDImageSeries( |
1060 | [(defer.FAILURE, Failure(exception))]) |
1061 | diff --git a/lib/lp/scripts/utilities/importpedant.py b/lib/lp/scripts/utilities/importpedant.py |
1062 | index ec41baf..c7859a4 100644 |
1063 | --- a/lib/lp/scripts/utilities/importpedant.py |
1064 | +++ b/lib/lp/scripts/utilities/importpedant.py |
1065 | @@ -1,4 +1,4 @@ |
1066 | -# Copyright 2009-2016 Canonical Ltd. This software is licensed under the |
1067 | +# Copyright 2009-2020 Canonical Ltd. This software is licensed under the |
1068 | # GNU Affero General Public License version 3 (see the file LICENSE). |
1069 | |
1070 | from __future__ import absolute_import, print_function, unicode_literals |
1071 | @@ -42,6 +42,8 @@ valid_imports_not_in_all = { |
1072 | 'textwrap': set(['dedent']), |
1073 | 'testtools.testresult.real': set(['_details_to_str']), |
1074 | 'twisted.internet.threads': set(['deferToThreadPool']), |
1075 | + # Even docs tell us to use this class. See docs on WebClientContextFactory. |
1076 | + 'twisted.web.client': set(['BrowserLikePolicyForHTTPS']), |
1077 | 'zope.component': set( |
1078 | ['adapter', |
1079 | 'ComponentLookupError', |
1080 | diff --git a/lib/lp/testing/factory.py b/lib/lp/testing/factory.py |
1081 | index 85a05b6..6c0492f 100644 |
1082 | --- a/lib/lp/testing/factory.py |
1083 | +++ b/lib/lp/testing/factory.py |
1084 | @@ -2,7 +2,7 @@ |
1085 | # NOTE: The first line above must stay first; do not move the copyright |
1086 | # notice to the top. See http://www.python.org/dev/peps/pep-0263/. |
1087 | # |
1088 | -# Copyright 2009-2019 Canonical Ltd. This software is licensed under the |
1089 | +# Copyright 2009-2020 Canonical Ltd. This software is licensed under the |
1090 | # GNU Affero General Public License version 3 (see the file LICENSE). |
1091 | |
1092 | """Testing infrastructure for the Launchpad application. |
1093 | @@ -3527,13 +3527,13 @@ class BareLaunchpadObjectFactory(ObjectFactory): |
1094 | return proberecord |
1095 | |
1096 | def makeMirror(self, distribution, displayname=None, country=None, |
1097 | - http_url=None, ftp_url=None, rsync_url=None, |
1098 | + http_url=None, https_url=None, ftp_url=None, rsync_url=None, |
1099 | official_candidate=False): |
1100 | """Create a mirror for the distribution.""" |
1101 | if displayname is None: |
1102 | displayname = self.getUniqueString("mirror") |
1103 | # If no URL is specified create an HTTP URL. |
1104 | - if http_url is None and ftp_url is None and rsync_url is None: |
1105 | + if http_url is https_url is ftp_url is rsync_url is None: |
1106 | http_url = self.getUniqueURL() |
1107 | # If no country is given use Argentina. |
1108 | if country is None: |
1109 | @@ -3547,6 +3547,7 @@ class BareLaunchpadObjectFactory(ObjectFactory): |
1110 | display_name=displayname, |
1111 | description=None, |
1112 | http_base_url=http_url, |
1113 | + https_base_url=https_url, |
1114 | ftp_base_url=ftp_url, |
1115 | rsync_base_url=rsync_url, |
1116 | official_candidate=official_candidate) |
Thanks. You can go ahead and re-land this now; the database patch has been deployed to production, and I've merged that revision of db-devel into master.