Merge ~pappacena/launchpad:revert-https-mirrors into launchpad:master
- Git
- lp:~pappacena/launchpad
- revert-https-mirrors
- Merge into master
Proposed by
Thiago F. Pappacena
Status: | Merged |
---|---|
Approved by: | Thiago F. Pappacena |
Approved revision: | feb7c661874d25353f24066d0e6df37e14a083a7 |
Merge reported by: | Otto Co-Pilot |
Merged at revision: | not available |
Proposed branch: | ~pappacena/launchpad:revert-https-mirrors |
Merge into: | launchpad:master |
Diff against target: |
1116 lines (+53/-477) 16 files modified
lib/lp/registry/browser/distributionmirror.py (+6/-7) lib/lp/registry/browser/tests/distributionmirror-views.txt (+9/-40) lib/lp/registry/configure.zcml (+3/-5) lib/lp/registry/interfaces/distribution.py (+4/-4) lib/lp/registry/interfaces/distributionmirror.py (+3/-21) lib/lp/registry/model/distribution.py (+4/-7) lib/lp/registry/model/distributionmirror.py (+2/-10) lib/lp/registry/scripts/distributionmirror_prober.py (+7/-153) lib/lp/registry/stories/webservice/xx-distribution-mirror.txt (+0/-4) lib/lp/registry/stories/webservice/xx-distribution.txt (+0/-1) lib/lp/registry/templates/distributionmirror-index.pt (+0/-4) lib/lp/registry/templates/distributionmirror-macros.pt (+1/-3) lib/lp/registry/tests/distributionmirror_http_server.py (+7/-15) lib/lp/registry/tests/test_distributionmirror_prober.py (+3/-196) lib/lp/scripts/utilities/importpedant.py (+1/-3) lib/lp/testing/factory.py (+3/-4) |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Thiago F. Pappacena (community) | Approve | ||
Review via email: mp+379909@code.launchpad.net |
Commit message
Reverting https-mirrors branch until we land the database patch.
Description of the change
To post a comment you must log in.
Revision history for this message
Otto Co-Pilot (otto-copilot) wrote : | # |
Revision history for this message
Thiago F. Pappacena (pappacena) : | # |
review:
Approve
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | diff --git a/lib/lp/registry/browser/distributionmirror.py b/lib/lp/registry/browser/distributionmirror.py | |||
2 | index 6ae1eb7..fd671a0 100644 | |||
3 | --- a/lib/lp/registry/browser/distributionmirror.py | |||
4 | +++ b/lib/lp/registry/browser/distributionmirror.py | |||
5 | @@ -1,4 +1,4 @@ | |||
7 | 1 | # Copyright 2009-2020 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009 Canonical Ltd. This software is licensed under the |
8 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
9 | 3 | 3 | ||
10 | 4 | __metaclass__ = type | 4 | __metaclass__ = type |
11 | @@ -207,9 +207,9 @@ class DistributionMirrorDeleteView(LaunchpadFormView): | |||
12 | 207 | class DistributionMirrorAddView(LaunchpadFormView): | 207 | class DistributionMirrorAddView(LaunchpadFormView): |
13 | 208 | schema = IDistributionMirror | 208 | schema = IDistributionMirror |
14 | 209 | field_names = [ | 209 | field_names = [ |
18 | 210 | "display_name", "description", "whiteboard", "https_base_url", | 210 | "display_name", "description", "whiteboard", "http_base_url", |
19 | 211 | "http_base_url", "ftp_base_url", "rsync_base_url", "speed", "country", | 211 | "ftp_base_url", "rsync_base_url", "speed", "country", "content", |
20 | 212 | "content", "official_candidate", | 212 | "official_candidate", |
21 | 213 | ] | 213 | ] |
22 | 214 | invariant_context = None | 214 | invariant_context = None |
23 | 215 | 215 | ||
24 | @@ -235,7 +235,6 @@ class DistributionMirrorAddView(LaunchpadFormView): | |||
25 | 235 | content=data['content'], display_name=data['display_name'], | 235 | content=data['content'], display_name=data['display_name'], |
26 | 236 | description=data['description'], | 236 | description=data['description'], |
27 | 237 | whiteboard=data['whiteboard'], | 237 | whiteboard=data['whiteboard'], |
28 | 238 | https_base_url=data['https_base_url'], | ||
29 | 239 | http_base_url=data['http_base_url'], | 238 | http_base_url=data['http_base_url'], |
30 | 240 | ftp_base_url=data['ftp_base_url'], | 239 | ftp_base_url=data['ftp_base_url'], |
31 | 241 | rsync_base_url=data['rsync_base_url'], | 240 | rsync_base_url=data['rsync_base_url'], |
32 | @@ -280,8 +279,8 @@ class DistributionMirrorEditView(LaunchpadEditFormView): | |||
33 | 280 | schema = IDistributionMirror | 279 | schema = IDistributionMirror |
34 | 281 | field_names = [ | 280 | field_names = [ |
35 | 282 | "name", "display_name", "description", "whiteboard", | 281 | "name", "display_name", "description", "whiteboard", |
38 | 283 | "https_base_url", "http_base_url", "ftp_base_url", "rsync_base_url", | 282 | "http_base_url", "ftp_base_url", "rsync_base_url", "speed", |
39 | 284 | "speed", "country", "content", "official_candidate", | 283 | "country", "content", "official_candidate", |
40 | 285 | ] | 284 | ] |
41 | 286 | 285 | ||
42 | 287 | @property | 286 | @property |
43 | diff --git a/lib/lp/registry/browser/tests/distributionmirror-views.txt b/lib/lp/registry/browser/tests/distributionmirror-views.txt | |||
44 | index f8fe4c5..e11dbd9 100644 | |||
45 | --- a/lib/lp/registry/browser/tests/distributionmirror-views.txt | |||
46 | +++ b/lib/lp/registry/browser/tests/distributionmirror-views.txt | |||
47 | @@ -44,19 +44,18 @@ The view provides a label, page_title, and cancel_url | |||
48 | 44 | >>> print view.cancel_url | 44 | >>> print view.cancel_url |
49 | 45 | http://launchpad.test/ubuntu | 45 | http://launchpad.test/ubuntu |
50 | 46 | 46 | ||
52 | 47 | A HTTP, HTTPS or FTP URL is required to register a mirror. | 47 | A HTTP or FTP URL is required to register a mirror. |
53 | 48 | 48 | ||
54 | 49 | >>> view.field_names | 49 | >>> view.field_names |
58 | 50 | ['display_name', 'description', 'whiteboard', 'https_base_url', | 50 | ['display_name', 'description', 'whiteboard', 'http_base_url', |
59 | 51 | 'http_base_url', 'ftp_base_url', 'rsync_base_url', 'speed', 'country', | 51 | 'ftp_base_url', 'rsync_base_url', 'speed', 'country', 'content', |
60 | 52 | 'content', 'official_candidate'] | 52 | 'official_candidate'] |
61 | 53 | 53 | ||
62 | 54 | >>> form = { | 54 | >>> form = { |
63 | 55 | ... 'field.display_name': 'Illuminati', | 55 | ... 'field.display_name': 'Illuminati', |
64 | 56 | ... 'field.description': 'description', | 56 | ... 'field.description': 'description', |
65 | 57 | ... 'field.whiteboard': 'whiteboard', | 57 | ... 'field.whiteboard': 'whiteboard', |
66 | 58 | ... 'field.http_base_url': 'http://secret.me/', | 58 | ... 'field.http_base_url': 'http://secret.me/', |
67 | 59 | ... 'field.https_base_url': '', | ||
68 | 60 | ... 'field.ftp_base_url': '', | 59 | ... 'field.ftp_base_url': '', |
69 | 61 | ... 'field.rsync_base_url': '', | 60 | ... 'field.rsync_base_url': '', |
70 | 62 | ... 'field.speed': 'S128K', | 61 | ... 'field.speed': 'S128K', |
71 | @@ -94,7 +93,6 @@ not significant). | |||
72 | 94 | The same is true for a FTP URL. | 93 | The same is true for a FTP URL. |
73 | 95 | 94 | ||
74 | 96 | >>> mirror.ftp_base_url = 'ftp://now-here.me/' | 95 | >>> mirror.ftp_base_url = 'ftp://now-here.me/' |
75 | 97 | >>> bad_form['field.https_base_url'] = '' | ||
76 | 98 | >>> bad_form['field.http_base_url'] = '' | 96 | >>> bad_form['field.http_base_url'] = '' |
77 | 99 | >>> bad_form['field.ftp_base_url'] = 'ftp://now-here.me' | 97 | >>> bad_form['field.ftp_base_url'] = 'ftp://now-here.me' |
78 | 100 | >>> view = create_initialized_view(ubuntu, '+newmirror', form=bad_form) | 98 | >>> view = create_initialized_view(ubuntu, '+newmirror', form=bad_form) |
79 | @@ -112,15 +110,14 @@ The same is true for a rsync URL. | |||
80 | 112 | ... print error[2] | 110 | ... print error[2] |
81 | 113 | The distribution mirror ... is already registered with this URL. | 111 | The distribution mirror ... is already registered with this URL. |
82 | 114 | 112 | ||
84 | 115 | A mirror must have an ftp, HTTPS or http URL. | 113 | A mirror must have an ftp or http URL. |
85 | 116 | 114 | ||
86 | 117 | >>> bad_form['field.https_base_url'] = '' | ||
87 | 118 | >>> bad_form['field.http_base_url'] = '' | 115 | >>> bad_form['field.http_base_url'] = '' |
88 | 119 | >>> bad_form['field.ftp_base_url'] = '' | 116 | >>> bad_form['field.ftp_base_url'] = '' |
89 | 120 | >>> view = create_initialized_view(ubuntu, '+newmirror', form=bad_form) | 117 | >>> view = create_initialized_view(ubuntu, '+newmirror', form=bad_form) |
90 | 121 | >>> for message in view.errors: | 118 | >>> for message in view.errors: |
91 | 122 | ... print message | 119 | ... print message |
93 | 123 | A mirror must have at least an HTTP(S) or FTP URL. | 120 | A mirror must have at least an HTTP or FTP URL. |
94 | 124 | 121 | ||
95 | 125 | The URL cannot contain a fragment. | 122 | The URL cannot contain a fragment. |
96 | 126 | 123 | ||
97 | @@ -138,34 +135,6 @@ The URL cannot contain a query string. | |||
98 | 138 | ... print error[2] | 135 | ... print error[2] |
99 | 139 | URIs with query strings are not allowed. | 136 | URIs with query strings are not allowed. |
100 | 140 | 137 | ||
101 | 141 | The HTTPS URL may not have an HTTP scheme. | ||
102 | 142 | |||
103 | 143 | >>> bad_form['field.http_base_url'] = '' | ||
104 | 144 | >>> bad_form['field.https_base_url'] = 'http://secret.me/#fragement' | ||
105 | 145 | >>> view = create_initialized_view(ubuntu, '+newmirror', form=bad_form) | ||
106 | 146 | >>> for error in view.errors: | ||
107 | 147 | ... print error[2] | ||
108 | 148 | The URI scheme "http" is not allowed. | ||
109 | 149 | Only URIs with the following schemes may be used: https | ||
110 | 150 | |||
111 | 151 | The HTTPS URL cannot contain a fragment. | ||
112 | 152 | |||
113 | 153 | >>> bad_form['field.http_base_url'] = '' | ||
114 | 154 | >>> bad_form['field.https_base_url'] = 'https://secret.me/#fragement' | ||
115 | 155 | >>> view = create_initialized_view(ubuntu, '+newmirror', form=bad_form) | ||
116 | 156 | >>> for error in view.errors: | ||
117 | 157 | ... print error[2] | ||
118 | 158 | URIs with fragment identifiers are not allowed. | ||
119 | 159 | |||
120 | 160 | The URL cannot contain a query string. | ||
121 | 161 | |||
122 | 162 | >>> bad_form['field.http_base_url'] = '' | ||
123 | 163 | >>> bad_form['field.https_base_url'] = 'https://secret.me/?query=string' | ||
124 | 164 | >>> view = create_initialized_view(ubuntu, '+newmirror', form=bad_form) | ||
125 | 165 | >>> for error in view.errors: | ||
126 | 166 | ... print error[2] | ||
127 | 167 | URIs with query strings are not allowed. | ||
128 | 168 | |||
129 | 169 | 138 | ||
130 | 170 | Reviewing a distribution mirror | 139 | Reviewing a distribution mirror |
131 | 171 | ------------------------------- | 140 | ------------------------------- |
132 | @@ -270,9 +239,9 @@ The +edit view provides a label, page_title, and cancel_url. | |||
133 | 270 | The user can edit the mirror fields. | 239 | The user can edit the mirror fields. |
134 | 271 | 240 | ||
135 | 272 | >>> view.field_names | 241 | >>> view.field_names |
139 | 273 | ['name', 'display_name', 'description', 'whiteboard', 'https_base_url', | 242 | ['name', 'display_name', 'description', 'whiteboard', 'http_base_url', |
140 | 274 | 'http_base_url', 'ftp_base_url', 'rsync_base_url', 'speed', 'country', | 243 | 'ftp_base_url', 'rsync_base_url', 'speed', 'country', 'content', |
141 | 275 | 'content', 'official_candidate'] | 244 | 'official_candidate'] |
142 | 276 | 245 | ||
143 | 277 | >>> print mirror.ftp_base_url | 246 | >>> print mirror.ftp_base_url |
144 | 278 | None | 247 | None |
145 | diff --git a/lib/lp/registry/configure.zcml b/lib/lp/registry/configure.zcml | |||
146 | index d5875c5..9be68e0 100644 | |||
147 | --- a/lib/lp/registry/configure.zcml | |||
148 | +++ b/lib/lp/registry/configure.zcml | |||
149 | @@ -1,4 +1,4 @@ | |||
151 | 1 | <!-- Copyright 2009-2020 Canonical Ltd. This software is licensed under the | 1 | <!-- Copyright 2009-2019 Canonical Ltd. This software is licensed under the |
152 | 2 | GNU Affero General Public License version 3 (see the file LICENSE). | 2 | GNU Affero General Public License version 3 (see the file LICENSE). |
153 | 3 | --> | 3 | --> |
154 | 4 | 4 | ||
155 | @@ -2046,7 +2046,6 @@ | |||
156 | 2046 | description | 2046 | description |
157 | 2047 | distribution | 2047 | distribution |
158 | 2048 | http_base_url | 2048 | http_base_url |
159 | 2049 | https_base_url | ||
160 | 2050 | ftp_base_url | 2049 | ftp_base_url |
161 | 2051 | rsync_base_url | 2050 | rsync_base_url |
162 | 2052 | enabled | 2051 | enabled |
163 | @@ -2085,9 +2084,8 @@ | |||
164 | 2085 | <require | 2084 | <require |
165 | 2086 | permission="launchpad.Edit" | 2085 | permission="launchpad.Edit" |
166 | 2087 | set_attributes="name display_name description whiteboard | 2086 | set_attributes="name display_name description whiteboard |
170 | 2088 | http_base_url https_base_url ftp_base_url | 2087 | http_base_url ftp_base_url rsync_base_url enabled |
171 | 2089 | rsync_base_url enabled speed country content | 2088 | speed country content official_candidate owner" |
169 | 2090 | official_candidate owner" | ||
172 | 2091 | attributes="official_candidate whiteboard resubmitForReview" /> | 2089 | attributes="official_candidate whiteboard resubmitForReview" /> |
173 | 2092 | <require | 2090 | <require |
174 | 2093 | permission="launchpad.Admin" | 2091 | permission="launchpad.Admin" |
175 | diff --git a/lib/lp/registry/interfaces/distribution.py b/lib/lp/registry/interfaces/distribution.py | |||
176 | index a7106f6..c7ddb95 100644 | |||
177 | --- a/lib/lp/registry/interfaces/distribution.py | |||
178 | +++ b/lib/lp/registry/interfaces/distribution.py | |||
179 | @@ -1,4 +1,4 @@ | |||
181 | 1 | # Copyright 2009-2020 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2015 Canonical Ltd. This software is licensed under the |
182 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
183 | 3 | 3 | ||
184 | 4 | """Interfaces including and related to IDistribution.""" | 4 | """Interfaces including and related to IDistribution.""" |
185 | @@ -491,13 +491,13 @@ class IDistributionPublic( | |||
186 | 491 | """Return the country DNS mirror for a country and content type.""" | 491 | """Return the country DNS mirror for a country and content type.""" |
187 | 492 | 492 | ||
188 | 493 | def newMirror(owner, speed, country, content, display_name=None, | 493 | def newMirror(owner, speed, country, content, display_name=None, |
190 | 494 | description=None, http_base_url=None, https_base_url=None, | 494 | description=None, http_base_url=None, |
191 | 495 | ftp_base_url=None, rsync_base_url=None, enabled=False, | 495 | ftp_base_url=None, rsync_base_url=None, enabled=False, |
192 | 496 | official_candidate=False, whiteboard=None): | 496 | official_candidate=False, whiteboard=None): |
193 | 497 | """Create a new DistributionMirror for this distribution. | 497 | """Create a new DistributionMirror for this distribution. |
194 | 498 | 498 | ||
197 | 499 | At least one of {http,https,ftp}_base_url must be provided in order to | 499 | At least one of http_base_url or ftp_base_url must be provided in |
198 | 500 | create a mirror. | 500 | order to create a mirror. |
199 | 501 | """ | 501 | """ |
200 | 502 | 502 | ||
201 | 503 | def getOCIProject(name): | 503 | def getOCIProject(name): |
202 | diff --git a/lib/lp/registry/interfaces/distributionmirror.py b/lib/lp/registry/interfaces/distributionmirror.py | |||
203 | index 4849e7a..ce6720e 100644 | |||
204 | --- a/lib/lp/registry/interfaces/distributionmirror.py | |||
205 | +++ b/lib/lp/registry/interfaces/distributionmirror.py | |||
206 | @@ -1,4 +1,4 @@ | |||
208 | 1 | # Copyright 2009-2020 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009 Canonical Ltd. This software is licensed under the |
209 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
210 | 3 | 3 | ||
211 | 4 | __metaclass__ = type | 4 | __metaclass__ = type |
212 | @@ -302,12 +302,6 @@ class DistroMirrorHTTPURIField(DistroMirrorURIField): | |||
213 | 302 | return getUtility(IDistributionMirrorSet).getByHttpUrl(url) | 302 | return getUtility(IDistributionMirrorSet).getByHttpUrl(url) |
214 | 303 | 303 | ||
215 | 304 | 304 | ||
216 | 305 | class DistroMirrorHTTPSURIField(DistroMirrorURIField): | ||
217 | 306 | |||
218 | 307 | def getMirrorByURI(self, url): | ||
219 | 308 | return getUtility(IDistributionMirrorSet).getByHttpsUrl(url) | ||
220 | 309 | |||
221 | 310 | |||
222 | 311 | class DistroMirrorFTPURIField(DistroMirrorURIField): | 305 | class DistroMirrorFTPURIField(DistroMirrorURIField): |
223 | 312 | 306 | ||
224 | 313 | def getMirrorByURI(self, url): | 307 | def getMirrorByURI(self, url): |
225 | @@ -355,14 +349,6 @@ class IDistributionMirror(Interface): | |||
226 | 355 | allowed_schemes=['http'], allow_userinfo=False, | 349 | allowed_schemes=['http'], allow_userinfo=False, |
227 | 356 | allow_query=False, allow_fragment=False, trailing_slash=True, | 350 | allow_query=False, allow_fragment=False, trailing_slash=True, |
228 | 357 | description=_('e.g.: http://archive.ubuntu.com/ubuntu/'))) | 351 | description=_('e.g.: http://archive.ubuntu.com/ubuntu/'))) |
229 | 358 | https_base_url = exported(DistroMirrorHTTPSURIField( | ||
230 | 359 | title=_('HTTPS URL'), required=False, readonly=False, | ||
231 | 360 | allowed_schemes=['https'], allow_userinfo=False, | ||
232 | 361 | allow_query=False, allow_fragment=False, trailing_slash=True, | ||
233 | 362 | # XXX: pappacena 2020-02-21: Add description field with a more | ||
234 | 363 | # suitable example once we have https for archive.ubuntu.com, like: | ||
235 | 364 | # description=_('e.g.: http://archive.ubuntu.com/ubuntu/') | ||
236 | 365 | )) | ||
237 | 366 | ftp_base_url = exported(DistroMirrorFTPURIField( | 352 | ftp_base_url = exported(DistroMirrorFTPURIField( |
238 | 367 | title=_('FTP URL'), required=False, readonly=False, | 353 | title=_('FTP URL'), required=False, readonly=False, |
239 | 368 | allowed_schemes=['ftp'], allow_userinfo=False, | 354 | allowed_schemes=['ftp'], allow_userinfo=False, |
240 | @@ -449,9 +435,8 @@ class IDistributionMirror(Interface): | |||
241 | 449 | 435 | ||
242 | 450 | @invariant | 436 | @invariant |
243 | 451 | def mirrorMustHaveHTTPOrFTPURL(mirror): | 437 | def mirrorMustHaveHTTPOrFTPURL(mirror): |
247 | 452 | if not (mirror.http_base_url or mirror.https_base_url or | 438 | if not (mirror.http_base_url or mirror.ftp_base_url): |
248 | 453 | mirror.ftp_base_url): | 439 | raise Invalid('A mirror must have at least an HTTP or FTP URL.') |
246 | 454 | raise Invalid('A mirror must have at least an HTTP(S) or FTP URL.') | ||
249 | 455 | 440 | ||
250 | 456 | def getSummarizedMirroredSourceSeries(): | 441 | def getSummarizedMirroredSourceSeries(): |
251 | 457 | """Return a summarized list of this distribution_mirror's | 442 | """Return a summarized list of this distribution_mirror's |
252 | @@ -629,9 +614,6 @@ class IDistributionMirrorSet(Interface): | |||
253 | 629 | def getByHttpUrl(url): | 614 | def getByHttpUrl(url): |
254 | 630 | """Return the mirror with the given HTTP URL or None.""" | 615 | """Return the mirror with the given HTTP URL or None.""" |
255 | 631 | 616 | ||
256 | 632 | def getByHttpsUrl(url): | ||
257 | 633 | """Return the mirror with the given HTTPS URL or None.""" | ||
258 | 634 | |||
259 | 635 | def getByFtpUrl(url): | 617 | def getByFtpUrl(url): |
260 | 636 | """Return the mirror with the given FTP URL or None.""" | 618 | """Return the mirror with the given FTP URL or None.""" |
261 | 637 | 619 | ||
262 | diff --git a/lib/lp/registry/model/distribution.py b/lib/lp/registry/model/distribution.py | |||
263 | index f78908b..2288d0e 100644 | |||
264 | --- a/lib/lp/registry/model/distribution.py | |||
265 | +++ b/lib/lp/registry/model/distribution.py | |||
266 | @@ -1,4 +1,4 @@ | |||
268 | 1 | # Copyright 2009-2020 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2016 Canonical Ltd. This software is licensed under the |
269 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
270 | 3 | 3 | ||
271 | 4 | """Database classes for implementing distribution items.""" | 4 | """Database classes for implementing distribution items.""" |
272 | @@ -709,7 +709,7 @@ class Distribution(SQLBase, BugTargetBase, MakesAnnouncements, | |||
273 | 709 | country_dns_mirror=True).one() | 709 | country_dns_mirror=True).one() |
274 | 710 | 710 | ||
275 | 711 | def newMirror(self, owner, speed, country, content, display_name=None, | 711 | def newMirror(self, owner, speed, country, content, display_name=None, |
277 | 712 | description=None, http_base_url=None, https_base_url=None, | 712 | description=None, http_base_url=None, |
278 | 713 | ftp_base_url=None, rsync_base_url=None, | 713 | ftp_base_url=None, rsync_base_url=None, |
279 | 714 | official_candidate=False, enabled=False, | 714 | official_candidate=False, enabled=False, |
280 | 715 | whiteboard=None): | 715 | whiteboard=None): |
281 | @@ -722,17 +722,15 @@ class Distribution(SQLBase, BugTargetBase, MakesAnnouncements, | |||
282 | 722 | return None | 722 | return None |
283 | 723 | 723 | ||
284 | 724 | urls = {'http_base_url': http_base_url, | 724 | urls = {'http_base_url': http_base_url, |
285 | 725 | 'https_base_url': https_base_url, | ||
286 | 726 | 'ftp_base_url': ftp_base_url, | 725 | 'ftp_base_url': ftp_base_url, |
287 | 727 | 'rsync_base_url': rsync_base_url} | 726 | 'rsync_base_url': rsync_base_url} |
288 | 728 | for name, value in urls.items(): | 727 | for name, value in urls.items(): |
289 | 729 | if value is not None: | 728 | if value is not None: |
290 | 730 | urls[name] = IDistributionMirror[name].normalize(value) | 729 | urls[name] = IDistributionMirror[name].normalize(value) |
291 | 731 | 730 | ||
294 | 732 | url = (urls['https_base_url'] or urls['http_base_url'] or | 731 | url = urls['http_base_url'] or urls['ftp_base_url'] |
293 | 733 | urls['ftp_base_url']) | ||
295 | 734 | assert url is not None, ( | 732 | assert url is not None, ( |
297 | 735 | "A mirror must provide at least one HTTP/HTTPS/FTP URL.") | 733 | "A mirror must provide either an HTTP or FTP URL (or both).") |
298 | 736 | dummy, host, dummy, dummy, dummy, dummy = urlparse(url) | 734 | dummy, host, dummy, dummy, dummy, dummy = urlparse(url) |
299 | 737 | name = sanitize_name('%s-%s' % (host, content.name.lower())) | 735 | name = sanitize_name('%s-%s' % (host, content.name.lower())) |
300 | 738 | 736 | ||
301 | @@ -746,7 +744,6 @@ class Distribution(SQLBase, BugTargetBase, MakesAnnouncements, | |||
302 | 746 | distribution=self, owner=owner, name=name, speed=speed, | 744 | distribution=self, owner=owner, name=name, speed=speed, |
303 | 747 | country=country, content=content, display_name=display_name, | 745 | country=country, content=content, display_name=display_name, |
304 | 748 | description=description, http_base_url=urls['http_base_url'], | 746 | description=description, http_base_url=urls['http_base_url'], |
305 | 749 | https_base_url=urls['https_base_url'], | ||
306 | 750 | ftp_base_url=urls['ftp_base_url'], | 747 | ftp_base_url=urls['ftp_base_url'], |
307 | 751 | rsync_base_url=urls['rsync_base_url'], | 748 | rsync_base_url=urls['rsync_base_url'], |
308 | 752 | official_candidate=official_candidate, enabled=enabled, | 749 | official_candidate=official_candidate, enabled=enabled, |
309 | diff --git a/lib/lp/registry/model/distributionmirror.py b/lib/lp/registry/model/distributionmirror.py | |||
310 | index 323fb26..747a2b0 100644 | |||
311 | --- a/lib/lp/registry/model/distributionmirror.py | |||
312 | +++ b/lib/lp/registry/model/distributionmirror.py | |||
313 | @@ -1,4 +1,4 @@ | |||
315 | 1 | # Copyright 2009-2020 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2015 Canonical Ltd. This software is licensed under the |
316 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
317 | 3 | 3 | ||
318 | 4 | """Module docstring goes here.""" | 4 | """Module docstring goes here.""" |
319 | @@ -129,8 +129,6 @@ class DistributionMirror(SQLBase): | |||
320 | 129 | notNull=False, default=None) | 129 | notNull=False, default=None) |
321 | 130 | http_base_url = StringCol( | 130 | http_base_url = StringCol( |
322 | 131 | notNull=False, default=None, unique=True) | 131 | notNull=False, default=None, unique=True) |
323 | 132 | https_base_url = StringCol( | ||
324 | 133 | notNull=False, default=None, unique=True) | ||
325 | 134 | ftp_base_url = StringCol( | 132 | ftp_base_url = StringCol( |
326 | 135 | notNull=False, default=None, unique=True) | 133 | notNull=False, default=None, unique=True) |
327 | 136 | rsync_base_url = StringCol( | 134 | rsync_base_url = StringCol( |
328 | @@ -157,9 +155,7 @@ class DistributionMirror(SQLBase): | |||
329 | 157 | @property | 155 | @property |
330 | 158 | def base_url(self): | 156 | def base_url(self): |
331 | 159 | """See IDistributionMirror""" | 157 | """See IDistributionMirror""" |
335 | 160 | if self.https_base_url is not None: | 158 | if self.http_base_url is not None: |
333 | 161 | return self.https_base_url | ||
334 | 162 | elif self.http_base_url is not None: | ||
336 | 163 | return self.http_base_url | 159 | return self.http_base_url |
337 | 164 | else: | 160 | else: |
338 | 165 | return self.ftp_base_url | 161 | return self.ftp_base_url |
339 | @@ -681,10 +677,6 @@ class DistributionMirrorSet: | |||
340 | 681 | """See IDistributionMirrorSet""" | 677 | """See IDistributionMirrorSet""" |
341 | 682 | return DistributionMirror.selectOneBy(http_base_url=url) | 678 | return DistributionMirror.selectOneBy(http_base_url=url) |
342 | 683 | 679 | ||
343 | 684 | def getByHttpsUrl(self, url): | ||
344 | 685 | """See IDistributionMirrorSet""" | ||
345 | 686 | return DistributionMirror.selectOneBy(https_base_url=url) | ||
346 | 687 | |||
347 | 688 | def getByFtpUrl(self, url): | 680 | def getByFtpUrl(self, url): |
348 | 689 | """See IDistributionMirrorSet""" | 681 | """See IDistributionMirrorSet""" |
349 | 690 | return DistributionMirror.selectOneBy(ftp_base_url=url) | 682 | return DistributionMirror.selectOneBy(ftp_base_url=url) |
350 | diff --git a/lib/lp/registry/scripts/distributionmirror_prober.py b/lib/lp/registry/scripts/distributionmirror_prober.py | |||
351 | index c633fb8..916d4ba 100644 | |||
352 | --- a/lib/lp/registry/scripts/distributionmirror_prober.py | |||
353 | +++ b/lib/lp/registry/scripts/distributionmirror_prober.py | |||
354 | @@ -1,4 +1,4 @@ | |||
356 | 1 | # Copyright 2009-2020 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2019 Canonical Ltd. This software is licensed under the |
357 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
358 | 3 | 3 | ||
359 | 4 | __metaclass__ = type | 4 | __metaclass__ = type |
360 | @@ -12,7 +12,6 @@ import logging | |||
361 | 12 | import os.path | 12 | import os.path |
362 | 13 | from StringIO import StringIO | 13 | from StringIO import StringIO |
363 | 14 | 14 | ||
364 | 15 | import OpenSSL | ||
365 | 16 | import requests | 15 | import requests |
366 | 17 | from six.moves import http_client | 16 | from six.moves import http_client |
367 | 18 | from six.moves.urllib.parse import ( | 17 | from six.moves.urllib.parse import ( |
368 | @@ -21,27 +20,14 @@ from six.moves.urllib.parse import ( | |||
369 | 21 | urlparse, | 20 | urlparse, |
370 | 22 | urlunparse, | 21 | urlunparse, |
371 | 23 | ) | 22 | ) |
372 | 24 | from treq.client import HTTPClient as TreqHTTPClient | ||
373 | 25 | from twisted.internet import ( | 23 | from twisted.internet import ( |
374 | 26 | defer, | 24 | defer, |
375 | 27 | protocol, | 25 | protocol, |
376 | 28 | reactor, | 26 | reactor, |
377 | 29 | ) | 27 | ) |
384 | 30 | from twisted.internet.defer import ( | 28 | from twisted.internet.defer import DeferredSemaphore |
379 | 31 | CancelledError, | ||
380 | 32 | DeferredSemaphore, | ||
381 | 33 | ) | ||
382 | 34 | from twisted.internet.endpoints import HostnameEndpoint | ||
383 | 35 | from twisted.internet.ssl import VerificationError | ||
385 | 36 | from twisted.python.failure import Failure | 29 | from twisted.python.failure import Failure |
386 | 37 | from twisted.web.client import ( | ||
387 | 38 | Agent, | ||
388 | 39 | BrowserLikePolicyForHTTPS, | ||
389 | 40 | ProxyAgent, | ||
390 | 41 | ResponseNeverReceived, | ||
391 | 42 | ) | ||
392 | 43 | from twisted.web.http import HTTPClient | 30 | from twisted.web.http import HTTPClient |
393 | 44 | from twisted.web.iweb import IResponse | ||
394 | 45 | from zope.component import getUtility | 31 | from zope.component import getUtility |
395 | 46 | 32 | ||
396 | 47 | from lp.app.interfaces.launchpad import ILaunchpadCelebrities | 33 | from lp.app.interfaces.launchpad import ILaunchpadCelebrities |
397 | @@ -64,7 +50,6 @@ from lp.soyuz.interfaces.distroarchseries import IDistroArchSeries | |||
398 | 64 | # IMPORTANT: Changing these values can cause lots of false negatives when | 50 | # IMPORTANT: Changing these values can cause lots of false negatives when |
399 | 65 | # probing mirrors, so please don't change them unless you know what you're | 51 | # probing mirrors, so please don't change them unless you know what you're |
400 | 66 | # doing. | 52 | # doing. |
401 | 67 | |||
402 | 68 | MIN_REQUEST_TIMEOUT_RATIO = 3 | 53 | MIN_REQUEST_TIMEOUT_RATIO = 3 |
403 | 69 | MIN_REQUESTS_TO_CONSIDER_RATIO = 30 | 54 | MIN_REQUESTS_TO_CONSIDER_RATIO = 30 |
404 | 70 | 55 | ||
405 | @@ -72,9 +57,6 @@ MIN_REQUESTS_TO_CONSIDER_RATIO = 30 | |||
406 | 72 | # We need to get rid of these global dicts in this module. | 57 | # We need to get rid of these global dicts in this module. |
407 | 73 | host_requests = {} | 58 | host_requests = {} |
408 | 74 | host_timeouts = {} | 59 | host_timeouts = {} |
409 | 75 | # Set of invalid certificate (host, port) tuples, to avoid doing HTTPS calls | ||
410 | 76 | # to hosts we already know they are not valid. | ||
411 | 77 | invalid_certificate_hosts = set() | ||
412 | 78 | 60 | ||
413 | 79 | MAX_REDIRECTS = 3 | 61 | MAX_REDIRECTS = 3 |
414 | 80 | 62 | ||
415 | @@ -179,64 +161,6 @@ class ProberProtocol(HTTPClient): | |||
416 | 179 | pass | 161 | pass |
417 | 180 | 162 | ||
418 | 181 | 163 | ||
419 | 182 | class HTTPSProbeFailureHandler: | ||
420 | 183 | """Handler to translate general errors into expected errors on HTTPS | ||
421 | 184 | connections.""" | ||
422 | 185 | def __init__(self, factory): | ||
423 | 186 | self.factory = factory | ||
424 | 187 | |||
425 | 188 | def handleResponse(self, response): | ||
426 | 189 | """Translates any request with return code different from 200 into | ||
427 | 190 | an error in the callback chain. | ||
428 | 191 | |||
429 | 192 | Note that other 2xx codes that are not 200 are considered errors too. | ||
430 | 193 | This behaviour is the same as seen in ProberProtocol.handleStatus, | ||
431 | 194 | for HTTP responses. | ||
432 | 195 | """ | ||
433 | 196 | status = response.code | ||
434 | 197 | if status == http_client.OK: | ||
435 | 198 | return response | ||
436 | 199 | else: | ||
437 | 200 | raise BadResponseCode(status, response) | ||
438 | 201 | |||
439 | 202 | def handleErrors(self, error): | ||
440 | 203 | """Handle exceptions in https requests. | ||
441 | 204 | """ | ||
442 | 205 | if self.isInvalidCertificateError(error): | ||
443 | 206 | invalid_certificate_hosts.add( | ||
444 | 207 | (self.factory.request_host, self.factory.request_port)) | ||
445 | 208 | reason = InvalidHTTPSCertificate( | ||
446 | 209 | self.factory.request_host, self.factory.request_port) | ||
447 | 210 | raise reason | ||
448 | 211 | if self.isTimeout(error): | ||
449 | 212 | raise ProberTimeout(self.factory.url, self.factory.timeout) | ||
450 | 213 | raise error | ||
451 | 214 | |||
452 | 215 | def isTimeout(self, error): | ||
453 | 216 | """Checks if the error was caused by a timeout. | ||
454 | 217 | """ | ||
455 | 218 | return self._isErrorFromType(error, CancelledError) | ||
456 | 219 | |||
457 | 220 | def isInvalidCertificateError(self, error): | ||
458 | 221 | """Checks if the error was caused by an invalid certificate. | ||
459 | 222 | """ | ||
460 | 223 | # It might be a raw SSL error, or a twisted-encapsulated | ||
461 | 224 | # verification error (such as DNSMismatch error when the | ||
462 | 225 | # certificate is valid for a different domain, for example). | ||
463 | 226 | return self._isErrorFromType( | ||
464 | 227 | error, OpenSSL.SSL.Error, VerificationError) | ||
465 | 228 | |||
466 | 229 | def _isErrorFromType(self, error, *types): | ||
467 | 230 | """Checks if the error was caused by any of the given types. | ||
468 | 231 | """ | ||
469 | 232 | if not isinstance(error.value, ResponseNeverReceived): | ||
470 | 233 | return False | ||
471 | 234 | for reason in error.value.reasons: | ||
472 | 235 | if reason.check(*types) is not None: | ||
473 | 236 | return True | ||
474 | 237 | return False | ||
475 | 238 | |||
476 | 239 | |||
477 | 240 | class RedirectAwareProberProtocol(ProberProtocol): | 164 | class RedirectAwareProberProtocol(ProberProtocol): |
478 | 241 | """A specialized version of ProberProtocol that follows HTTP redirects.""" | 165 | """A specialized version of ProberProtocol that follows HTTP redirects.""" |
479 | 242 | 166 | ||
480 | @@ -292,8 +216,6 @@ class ProberFactory(protocol.ClientFactory): | |||
481 | 292 | connect_port = None | 216 | connect_port = None |
482 | 293 | connect_path = None | 217 | connect_path = None |
483 | 294 | 218 | ||
484 | 295 | https_agent_policy = BrowserLikePolicyForHTTPS | ||
485 | 296 | |||
486 | 297 | def __init__(self, url, timeout=config.distributionmirrorprober.timeout): | 219 | def __init__(self, url, timeout=config.distributionmirrorprober.timeout): |
487 | 298 | # We want the deferred to be a private attribute (_deferred) to make | 220 | # We want the deferred to be a private attribute (_deferred) to make |
488 | 299 | # sure our clients will only use the deferred returned by the probe() | 221 | # sure our clients will only use the deferred returned by the probe() |
489 | @@ -303,14 +225,9 @@ class ProberFactory(protocol.ClientFactory): | |||
490 | 303 | self.timeout = timeout | 225 | self.timeout = timeout |
491 | 304 | self.timeoutCall = None | 226 | self.timeoutCall = None |
492 | 305 | self.setURL(url.encode('ascii')) | 227 | self.setURL(url.encode('ascii')) |
493 | 306 | self.logger = logging.getLogger('distributionmirror-prober') | ||
494 | 307 | |||
495 | 308 | @property | ||
496 | 309 | def is_https(self): | ||
497 | 310 | return self.request_scheme == 'https' | ||
498 | 311 | 228 | ||
499 | 312 | def probe(self): | 229 | def probe(self): |
501 | 313 | logger = self.logger | 230 | logger = logging.getLogger('distributionmirror-prober') |
502 | 314 | # NOTE: We don't want to issue connections to any outside host when | 231 | # NOTE: We don't want to issue connections to any outside host when |
503 | 315 | # running the mirror prober in a development machine, so we do this | 232 | # running the mirror prober in a development machine, so we do this |
504 | 316 | # hack here. | 233 | # hack here. |
505 | @@ -327,46 +244,13 @@ class ProberFactory(protocol.ClientFactory): | |||
506 | 327 | "host already." % self.url) | 244 | "host already." % self.url) |
507 | 328 | return self._deferred | 245 | return self._deferred |
508 | 329 | 246 | ||
509 | 330 | if (self.request_host, self.request_port) in invalid_certificate_hosts: | ||
510 | 331 | reactor.callLater( | ||
511 | 332 | 0, self.failed, InvalidHTTPSCertificateSkipped(self.url)) | ||
512 | 333 | logger.debug("Skipping %s as it doesn't have a valid HTTPS " | ||
513 | 334 | "certificate" % self.url) | ||
514 | 335 | return self._deferred | ||
515 | 336 | |||
516 | 337 | self.connect() | 247 | self.connect() |
517 | 338 | logger.debug('Probing %s' % self.url) | 248 | logger.debug('Probing %s' % self.url) |
518 | 339 | return self._deferred | 249 | return self._deferred |
519 | 340 | 250 | ||
520 | 341 | def getHttpsClient(self): | ||
521 | 342 | # Should we use a proxy? | ||
522 | 343 | if not config.launchpad.http_proxy: | ||
523 | 344 | agent = Agent( | ||
524 | 345 | reactor=reactor, contextFactory=self.https_agent_policy()) | ||
525 | 346 | else: | ||
526 | 347 | endpoint = HostnameEndpoint( | ||
527 | 348 | reactor, self.connect_host, self.connect_port) | ||
528 | 349 | agent = ProxyAgent(endpoint) | ||
529 | 350 | return TreqHTTPClient(agent) | ||
530 | 351 | |||
531 | 352 | def connect(self): | 251 | def connect(self): |
532 | 353 | """Starts the connection and sets the self._deferred to the proper | ||
533 | 354 | task. | ||
534 | 355 | """ | ||
535 | 356 | host_requests[self.request_host] += 1 | 252 | host_requests[self.request_host] += 1 |
549 | 357 | if self.is_https: | 253 | reactor.connectTCP(self.connect_host, self.connect_port, self) |
537 | 358 | treq = self.getHttpsClient() | ||
538 | 359 | self._deferred.addCallback( | ||
539 | 360 | lambda _: treq.head( | ||
540 | 361 | self.url, reactor=reactor, allow_redirects=True, | ||
541 | 362 | timeout=self.timeout)) | ||
542 | 363 | error_handler = HTTPSProbeFailureHandler(self) | ||
543 | 364 | self._deferred.addCallback(error_handler.handleResponse) | ||
544 | 365 | self._deferred.addErrback(error_handler.handleErrors) | ||
545 | 366 | reactor.callWhenRunning(self._deferred.callback, None) | ||
546 | 367 | else: | ||
547 | 368 | reactor.connectTCP(self.connect_host, self.connect_port, self) | ||
548 | 369 | |||
550 | 370 | if self.timeoutCall is not None and self.timeoutCall.active(): | 254 | if self.timeoutCall is not None and self.timeoutCall.active(): |
551 | 371 | self._cancelTimeout(None) | 255 | self._cancelTimeout(None) |
552 | 372 | self.timeoutCall = reactor.callLater( | 256 | self.timeoutCall = reactor.callLater( |
553 | @@ -385,8 +269,6 @@ class ProberFactory(protocol.ClientFactory): | |||
554 | 385 | self.connector = connector | 269 | self.connector = connector |
555 | 386 | 270 | ||
556 | 387 | def succeeded(self, status): | 271 | def succeeded(self, status): |
557 | 388 | if IResponse.providedBy(status): | ||
558 | 389 | status = str(status.code) | ||
559 | 390 | self._deferred.callback(status) | 272 | self._deferred.callback(status) |
560 | 391 | 273 | ||
561 | 392 | def failed(self, reason): | 274 | def failed(self, reason): |
562 | @@ -406,7 +288,7 @@ class ProberFactory(protocol.ClientFactory): | |||
563 | 406 | # https://bugs.squid-cache.org/show_bug.cgi?id=1758 applied. So, if | 288 | # https://bugs.squid-cache.org/show_bug.cgi?id=1758 applied. So, if |
564 | 407 | # you encounter any problems with FTP URLs you'll probably have to nag | 289 | # you encounter any problems with FTP URLs you'll probably have to nag |
565 | 408 | # the sysadmins to fix squid for you. | 290 | # the sysadmins to fix squid for you. |
567 | 409 | if scheme not in ('http', 'https', 'ftp'): | 291 | if scheme not in ('http', 'ftp'): |
568 | 410 | raise UnknownURLScheme(url) | 292 | raise UnknownURLScheme(url) |
569 | 411 | 293 | ||
570 | 412 | if scheme and host: | 294 | if scheme and host: |
571 | @@ -494,26 +376,14 @@ class ProberTimeout(ProberError): | |||
572 | 494 | 376 | ||
573 | 495 | class BadResponseCode(ProberError): | 377 | class BadResponseCode(ProberError): |
574 | 496 | 378 | ||
576 | 497 | def __init__(self, status, response=None, *args): | 379 | def __init__(self, status, *args): |
577 | 498 | ProberError.__init__(self, *args) | 380 | ProberError.__init__(self, *args) |
578 | 499 | self.status = status | 381 | self.status = status |
579 | 500 | self.response = response | ||
580 | 501 | 382 | ||
581 | 502 | def __str__(self): | 383 | def __str__(self): |
582 | 503 | return "Bad response code: %s" % self.status | 384 | return "Bad response code: %s" % self.status |
583 | 504 | 385 | ||
584 | 505 | 386 | ||
585 | 506 | class InvalidHTTPSCertificate(ProberError): | ||
586 | 507 | def __init__(self, host, port, *args): | ||
587 | 508 | super(InvalidHTTPSCertificate, self).__init__(*args) | ||
588 | 509 | self.host = host | ||
589 | 510 | self.port = port | ||
590 | 511 | |||
591 | 512 | def __str__(self): | ||
592 | 513 | return "Invalid SSL certificate when trying to probe %s:%s" % ( | ||
593 | 514 | self.host, self.port) | ||
594 | 515 | |||
595 | 516 | |||
596 | 517 | class RedirectToDifferentFile(ProberError): | 387 | class RedirectToDifferentFile(ProberError): |
597 | 518 | 388 | ||
598 | 519 | def __init__(self, orig_path, new_path, *args): | 389 | def __init__(self, orig_path, new_path, *args): |
599 | @@ -539,14 +409,6 @@ class ConnectionSkipped(ProberError): | |||
600 | 539 | "host. It will be retried on the next probing run.") | 409 | "host. It will be retried on the next probing run.") |
601 | 540 | 410 | ||
602 | 541 | 411 | ||
603 | 542 | class InvalidHTTPSCertificateSkipped(ProberError): | ||
604 | 543 | |||
605 | 544 | def __str__(self): | ||
606 | 545 | return ("Connection skipped because the server doesn't have a valid " | ||
607 | 546 | "HTTPS certificate. It will be retried on the next " | ||
608 | 547 | "probing run.") | ||
609 | 548 | |||
610 | 549 | |||
611 | 550 | class UnknownURLScheme(ProberError): | 412 | class UnknownURLScheme(ProberError): |
612 | 551 | 413 | ||
613 | 552 | def __init__(self, url, *args): | 414 | def __init__(self, url, *args): |
614 | @@ -567,13 +429,7 @@ class UnknownURLSchemeAfterRedirect(UnknownURLScheme): | |||
615 | 567 | 429 | ||
616 | 568 | class ArchiveMirrorProberCallbacks(LoggingMixin): | 430 | class ArchiveMirrorProberCallbacks(LoggingMixin): |
617 | 569 | 431 | ||
625 | 570 | expected_failures = ( | 432 | expected_failures = (BadResponseCode, ProberTimeout, ConnectionSkipped) |
619 | 571 | BadResponseCode, | ||
620 | 572 | ProberTimeout, | ||
621 | 573 | ConnectionSkipped, | ||
622 | 574 | InvalidHTTPSCertificate, | ||
623 | 575 | InvalidHTTPSCertificateSkipped, | ||
624 | 576 | ) | ||
626 | 577 | 433 | ||
627 | 578 | def __init__(self, mirror, series, pocket, component, url, log_file): | 434 | def __init__(self, mirror, series, pocket, component, url, log_file): |
628 | 579 | self.mirror = mirror | 435 | self.mirror = mirror |
629 | @@ -718,8 +574,6 @@ class MirrorCDImageProberCallbacks(LoggingMixin): | |||
630 | 718 | ProberTimeout, | 574 | ProberTimeout, |
631 | 719 | RedirectToDifferentFile, | 575 | RedirectToDifferentFile, |
632 | 720 | UnknownURLSchemeAfterRedirect, | 576 | UnknownURLSchemeAfterRedirect, |
633 | 721 | InvalidHTTPSCertificate, | ||
634 | 722 | InvalidHTTPSCertificateSkipped, | ||
635 | 723 | ) | 577 | ) |
636 | 724 | 578 | ||
637 | 725 | def __init__(self, mirror, distroseries, flavour, log_file): | 579 | def __init__(self, mirror, distroseries, flavour, log_file): |
638 | diff --git a/lib/lp/registry/stories/webservice/xx-distribution-mirror.txt b/lib/lp/registry/stories/webservice/xx-distribution-mirror.txt | |||
639 | index 3e49ebe..db17b40 100644 | |||
640 | --- a/lib/lp/registry/stories/webservice/xx-distribution-mirror.txt | |||
641 | +++ b/lib/lp/registry/stories/webservice/xx-distribution-mirror.txt | |||
642 | @@ -23,7 +23,6 @@ mirrors: | |||
643 | 23 | enabled: True | 23 | enabled: True |
644 | 24 | ftp_base_url: None | 24 | ftp_base_url: None |
645 | 25 | http_base_url: u'http://archive.ubuntu.com/ubuntu/' | 25 | http_base_url: u'http://archive.ubuntu.com/ubuntu/' |
646 | 26 | https_base_url: None | ||
647 | 27 | name: u'canonical-archive' | 26 | name: u'canonical-archive' |
648 | 28 | official_candidate: True | 27 | official_candidate: True |
649 | 29 | owner_link: u'http://.../~mark' | 28 | owner_link: u'http://.../~mark' |
650 | @@ -53,7 +52,6 @@ And CD image mirrors: | |||
651 | 53 | enabled: True | 52 | enabled: True |
652 | 54 | ftp_base_url: None | 53 | ftp_base_url: None |
653 | 55 | http_base_url: u'http://releases.ubuntu.com/' | 54 | http_base_url: u'http://releases.ubuntu.com/' |
654 | 56 | https_base_url: None | ||
655 | 57 | name: u'canonical-releases' | 55 | name: u'canonical-releases' |
656 | 58 | official_candidate: True | 56 | official_candidate: True |
657 | 59 | owner_link: u'http://.../~mark' | 57 | owner_link: u'http://.../~mark' |
658 | @@ -147,7 +145,6 @@ Mirror listing admins may see all: | |||
659 | 147 | enabled: True | 145 | enabled: True |
660 | 148 | ftp_base_url: None | 146 | ftp_base_url: None |
661 | 149 | http_base_url: u'http://localhost:11375/archive-mirror/' | 147 | http_base_url: u'http://localhost:11375/archive-mirror/' |
662 | 150 | https_base_url: None | ||
663 | 151 | name: u'archive-404-mirror' | 148 | name: u'archive-404-mirror' |
664 | 152 | official_candidate: True | 149 | official_candidate: True |
665 | 153 | owner_link: u'http://.../~name12' | 150 | owner_link: u'http://.../~name12' |
666 | @@ -230,7 +227,6 @@ While others can be set with the appropriate authorization: | |||
667 | 230 | enabled: True | 227 | enabled: True |
668 | 231 | ftp_base_url: None | 228 | ftp_base_url: None |
669 | 232 | http_base_url: u'http://releases.ubuntu.com/' | 229 | http_base_url: u'http://releases.ubuntu.com/' |
670 | 233 | https_base_url: None | ||
671 | 234 | name: u'canonical-releases' | 230 | name: u'canonical-releases' |
672 | 235 | official_candidate: True | 231 | official_candidate: True |
673 | 236 | owner_link: u'http://.../~mark' | 232 | owner_link: u'http://.../~mark' |
674 | diff --git a/lib/lp/registry/stories/webservice/xx-distribution.txt b/lib/lp/registry/stories/webservice/xx-distribution.txt | |||
675 | index 1486904..1c0ed87 100644 | |||
676 | --- a/lib/lp/registry/stories/webservice/xx-distribution.txt | |||
677 | +++ b/lib/lp/registry/stories/webservice/xx-distribution.txt | |||
678 | @@ -159,7 +159,6 @@ packages matching (substring) the given text. | |||
679 | 159 | enabled: True | 159 | enabled: True |
680 | 160 | ftp_base_url: None | 160 | ftp_base_url: None |
681 | 161 | http_base_url: u'http://releases.ubuntu.com/' | 161 | http_base_url: u'http://releases.ubuntu.com/' |
682 | 162 | https_base_url: None | ||
683 | 163 | name: u'canonical-releases' | 162 | name: u'canonical-releases' |
684 | 164 | official_candidate: True | 163 | official_candidate: True |
685 | 165 | owner_link: u'http://.../~mark' | 164 | owner_link: u'http://.../~mark' |
686 | diff --git a/lib/lp/registry/templates/distributionmirror-index.pt b/lib/lp/registry/templates/distributionmirror-index.pt | |||
687 | index 27f3139..78e934d 100644 | |||
688 | --- a/lib/lp/registry/templates/distributionmirror-index.pt | |||
689 | +++ b/lib/lp/registry/templates/distributionmirror-index.pt | |||
690 | @@ -118,10 +118,6 @@ | |||
691 | 118 | <h2>Mirror location information</h2> | 118 | <h2>Mirror location information</h2> |
692 | 119 | 119 | ||
693 | 120 | <ul class="webref" id="mirror-urls"> | 120 | <ul class="webref" id="mirror-urls"> |
694 | 121 | <li tal:condition="context/https_base_url" > | ||
695 | 122 | <a tal:content="context/https_base_url" | ||
696 | 123 | tal:attributes="href context/https_base_url">https://url/</a> | ||
697 | 124 | </li> | ||
698 | 125 | <li tal:condition="context/http_base_url" > | 121 | <li tal:condition="context/http_base_url" > |
699 | 126 | <a tal:content="context/http_base_url" | 122 | <a tal:content="context/http_base_url" |
700 | 127 | tal:attributes="href context/http_base_url">http://url/</a> | 123 | tal:attributes="href context/http_base_url">http://url/</a> |
701 | diff --git a/lib/lp/registry/templates/distributionmirror-macros.pt b/lib/lp/registry/templates/distributionmirror-macros.pt | |||
702 | index 489f770..18f82af 100644 | |||
703 | --- a/lib/lp/registry/templates/distributionmirror-macros.pt | |||
704 | +++ b/lib/lp/registry/templates/distributionmirror-macros.pt | |||
705 | @@ -17,7 +17,7 @@ | |||
706 | 17 | <tbody> | 17 | <tbody> |
707 | 18 | <tal:country_and_mirrors repeat="country_and_mirrors mirrors_by_country"> | 18 | <tal:country_and_mirrors repeat="country_and_mirrors mirrors_by_country"> |
708 | 19 | <tr class="head"> | 19 | <tr class="head"> |
710 | 20 | <th colspan="2" | 20 | <th colspan="2" |
711 | 21 | tal:content="country_and_mirrors/country" /> | 21 | tal:content="country_and_mirrors/country" /> |
712 | 22 | <th tal:content="country_and_mirrors/throughput"/> | 22 | <th tal:content="country_and_mirrors/throughput"/> |
713 | 23 | <th tal:condition="show_mirror_type"> | 23 | <th tal:condition="show_mirror_type"> |
714 | @@ -35,8 +35,6 @@ | |||
715 | 35 | tal:content="mirror/title">Mirror Name</a> | 35 | tal:content="mirror/title">Mirror Name</a> |
716 | 36 | </td> | 36 | </td> |
717 | 37 | <td> | 37 | <td> |
718 | 38 | <a tal:condition="mirror/https_base_url" | ||
719 | 39 | tal:attributes="href mirror/https_base_url">https</a> | ||
720 | 40 | <a tal:condition="mirror/http_base_url" | 38 | <a tal:condition="mirror/http_base_url" |
721 | 41 | tal:attributes="href mirror/http_base_url">http</a> | 39 | tal:attributes="href mirror/http_base_url">http</a> |
722 | 42 | <a tal:condition="mirror/ftp_base_url" | 40 | <a tal:condition="mirror/ftp_base_url" |
723 | diff --git a/lib/lp/registry/tests/distributionmirror_http_server.py b/lib/lp/registry/tests/distributionmirror_http_server.py | |||
724 | index d12c9b3..e3e0a39 100644 | |||
725 | --- a/lib/lp/registry/tests/distributionmirror_http_server.py | |||
726 | +++ b/lib/lp/registry/tests/distributionmirror_http_server.py | |||
727 | @@ -1,6 +1,6 @@ | |||
728 | 1 | #!/usr/bin/python | 1 | #!/usr/bin/python |
729 | 2 | # | 2 | # |
731 | 3 | # Copyright 2009-2020 Canonical Ltd. This software is licensed under the | 3 | # Copyright 2009 Canonical Ltd. This software is licensed under the |
732 | 4 | # GNU Affero General Public License version 3 (see the file LICENSE). | 4 | # GNU Affero General Public License version 3 (see the file LICENSE). |
733 | 5 | 5 | ||
734 | 6 | from twisted.web.resource import Resource | 6 | from twisted.web.resource import Resource |
735 | @@ -21,10 +21,10 @@ class DistributionMirrorTestHTTPServer(Resource): | |||
736 | 21 | :error: Respond with a '500 Internal Server Error' status. | 21 | :error: Respond with a '500 Internal Server Error' status. |
737 | 22 | 22 | ||
738 | 23 | :redirect-to-valid-mirror/*: Respond with a '302 Found' status, | 23 | :redirect-to-valid-mirror/*: Respond with a '302 Found' status, |
740 | 24 | redirecting to http(s)://localhost:%(port)s/valid-mirror/*. | 24 | redirecting to http://localhost:%(port)s/valid-mirror/*. |
741 | 25 | 25 | ||
742 | 26 | :redirect-infinite-loop: Respond with a '302 Found' status, redirecting | 26 | :redirect-infinite-loop: Respond with a '302 Found' status, redirecting |
744 | 27 | to http(s)://localhost:%(port)s/redirect-infinite-loop. | 27 | to http://localhost:%(port)s/redirect-infinite-loop. |
745 | 28 | 28 | ||
746 | 29 | :redirect-unknown-url-scheme: Respond with a '302 Found' status, | 29 | :redirect-unknown-url-scheme: Respond with a '302 Found' status, |
747 | 30 | redirecting to ssh://localhost/redirect-unknown-url-scheme. | 30 | redirecting to ssh://localhost/redirect-unknown-url-scheme. |
748 | @@ -32,13 +32,11 @@ class DistributionMirrorTestHTTPServer(Resource): | |||
749 | 32 | Any other path will cause the server to respond with a '404 Not Found' | 32 | Any other path will cause the server to respond with a '404 Not Found' |
750 | 33 | status. | 33 | status. |
751 | 34 | """ | 34 | """ |
752 | 35 | protocol = "http" | ||
753 | 36 | 35 | ||
754 | 37 | def getChild(self, name, request): | 36 | def getChild(self, name, request): |
755 | 38 | protocol = self.protocol | ||
756 | 39 | port = request.getHost().port | 37 | port = request.getHost().port |
757 | 40 | if name == 'valid-mirror': | 38 | if name == 'valid-mirror': |
759 | 41 | leaf = self.__class__() | 39 | leaf = DistributionMirrorTestHTTPServer() |
760 | 42 | leaf.isLeaf = True | 40 | leaf.isLeaf = True |
761 | 43 | return leaf | 41 | return leaf |
762 | 44 | elif name == 'timeout': | 42 | elif name == 'timeout': |
763 | @@ -51,14 +49,12 @@ class DistributionMirrorTestHTTPServer(Resource): | |||
764 | 51 | 'than one component.') | 49 | 'than one component.') |
765 | 52 | remaining_path = request.path.replace('/%s' % name, '') | 50 | remaining_path = request.path.replace('/%s' % name, '') |
766 | 53 | leaf = RedirectingResource( | 51 | leaf = RedirectingResource( |
769 | 54 | '%s://localhost:%s/valid-mirror%s' % ( | 52 | 'http://localhost:%s/valid-mirror%s' % (port, remaining_path)) |
768 | 55 | protocol, port, remaining_path)) | ||
770 | 56 | leaf.isLeaf = True | 53 | leaf.isLeaf = True |
771 | 57 | return leaf | 54 | return leaf |
772 | 58 | elif name == 'redirect-infinite-loop': | 55 | elif name == 'redirect-infinite-loop': |
773 | 59 | return RedirectingResource( | 56 | return RedirectingResource( |
776 | 60 | '%s://localhost:%s/redirect-infinite-loop' % | 57 | 'http://localhost:%s/redirect-infinite-loop' % port) |
775 | 61 | (protocol, port)) | ||
777 | 62 | elif name == 'redirect-unknown-url-scheme': | 58 | elif name == 'redirect-unknown-url-scheme': |
778 | 63 | return RedirectingResource( | 59 | return RedirectingResource( |
779 | 64 | 'ssh://localhost/redirect-unknown-url-scheme') | 60 | 'ssh://localhost/redirect-unknown-url-scheme') |
780 | @@ -69,11 +65,6 @@ class DistributionMirrorTestHTTPServer(Resource): | |||
781 | 69 | return "Hi" | 65 | return "Hi" |
782 | 70 | 66 | ||
783 | 71 | 67 | ||
784 | 72 | class DistributionMirrorTestSecureHTTPServer(DistributionMirrorTestHTTPServer): | ||
785 | 73 | """HTTPS version of DistributionMirrorTestHTTPServer""" | ||
786 | 74 | protocol = "https" | ||
787 | 75 | |||
788 | 76 | |||
789 | 77 | class RedirectingResource(Resource): | 68 | class RedirectingResource(Resource): |
790 | 78 | 69 | ||
791 | 79 | def __init__(self, redirection_url): | 70 | def __init__(self, redirection_url): |
792 | @@ -94,3 +85,4 @@ class FiveHundredResource(Resource): | |||
793 | 94 | def render_GET(self, request): | 85 | def render_GET(self, request): |
794 | 95 | request.setResponseCode(500) | 86 | request.setResponseCode(500) |
795 | 96 | request.write('ASPLODE!!!') | 87 | request.write('ASPLODE!!!') |
796 | 88 | return NOT_DONE_YET | ||
797 | diff --git a/lib/lp/registry/tests/test_distributionmirror_prober.py b/lib/lp/registry/tests/test_distributionmirror_prober.py | |||
798 | index c3e363c..39228d2 100644 | |||
799 | --- a/lib/lp/registry/tests/test_distributionmirror_prober.py | |||
800 | +++ b/lib/lp/registry/tests/test_distributionmirror_prober.py | |||
801 | @@ -1,4 +1,4 @@ | |||
803 | 1 | # Copyright 2009-2020 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2018 Canonical Ltd. This software is licensed under the |
804 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
805 | 3 | 3 | ||
806 | 4 | """distributionmirror-prober tests.""" | 4 | """distributionmirror-prober tests.""" |
807 | @@ -11,7 +11,6 @@ import logging | |||
808 | 11 | import os | 11 | import os |
809 | 12 | from StringIO import StringIO | 12 | from StringIO import StringIO |
810 | 13 | 13 | ||
811 | 14 | from fixtures import MockPatchObject | ||
812 | 15 | from lazr.uri import URI | 14 | from lazr.uri import URI |
813 | 16 | import responses | 15 | import responses |
814 | 17 | from six.moves import http_client | 16 | from six.moves import http_client |
815 | @@ -29,14 +28,9 @@ from testtools.twistedsupport import ( | |||
816 | 29 | from twisted.internet import ( | 28 | from twisted.internet import ( |
817 | 30 | defer, | 29 | defer, |
818 | 31 | reactor, | 30 | reactor, |
819 | 32 | ssl, | ||
820 | 33 | ) | 31 | ) |
821 | 34 | from twisted.python.failure import Failure | 32 | from twisted.python.failure import Failure |
822 | 35 | from twisted.web import server | 33 | from twisted.web import server |
823 | 36 | from twisted.web.client import ( | ||
824 | 37 | BrowserLikePolicyForHTTPS, | ||
825 | 38 | ProxyAgent, | ||
826 | 39 | ) | ||
827 | 40 | from zope.component import getUtility | 34 | from zope.component import getUtility |
828 | 41 | from zope.security.proxy import removeSecurityProxy | 35 | from zope.security.proxy import removeSecurityProxy |
829 | 42 | 36 | ||
830 | @@ -50,8 +44,6 @@ from lp.registry.scripts.distributionmirror_prober import ( | |||
831 | 50 | BadResponseCode, | 44 | BadResponseCode, |
832 | 51 | ConnectionSkipped, | 45 | ConnectionSkipped, |
833 | 52 | InfiniteLoopDetected, | 46 | InfiniteLoopDetected, |
834 | 53 | InvalidHTTPSCertificate, | ||
835 | 54 | InvalidHTTPSCertificateSkipped, | ||
836 | 55 | LoggingMixin, | 47 | LoggingMixin, |
837 | 56 | MAX_REDIRECTS, | 48 | MAX_REDIRECTS, |
838 | 57 | MIN_REQUEST_TIMEOUT_RATIO, | 49 | MIN_REQUEST_TIMEOUT_RATIO, |
839 | @@ -73,7 +65,6 @@ from lp.registry.scripts.distributionmirror_prober import ( | |||
840 | 73 | ) | 65 | ) |
841 | 74 | from lp.registry.tests.distributionmirror_http_server import ( | 66 | from lp.registry.tests.distributionmirror_http_server import ( |
842 | 75 | DistributionMirrorTestHTTPServer, | 67 | DistributionMirrorTestHTTPServer, |
843 | 76 | DistributionMirrorTestSecureHTTPServer, | ||
844 | 77 | ) | 68 | ) |
845 | 78 | from lp.services.config import config | 69 | from lp.services.config import config |
846 | 79 | from lp.services.daemons.tachandler import TacTestSetup | 70 | from lp.services.daemons.tachandler import TacTestSetup |
847 | @@ -112,186 +103,6 @@ class HTTPServerTestSetup(TacTestSetup): | |||
848 | 112 | return os.path.join(self.root, 'distributionmirror_http_server.log') | 103 | return os.path.join(self.root, 'distributionmirror_http_server.log') |
849 | 113 | 104 | ||
850 | 114 | 105 | ||
851 | 115 | |||
852 | 116 | class LocalhostWhitelistedHTTPSPolicy(BrowserLikePolicyForHTTPS): | ||
853 | 117 | """HTTPS policy that bypasses SSL certificate check when doing requests | ||
854 | 118 | to localhost. | ||
855 | 119 | """ | ||
856 | 120 | |||
857 | 121 | def creatorForNetloc(self, hostname, port): | ||
858 | 122 | # check if the hostname is in the the whitelist, | ||
859 | 123 | # otherwise return the default policy | ||
860 | 124 | if hostname == 'localhost': | ||
861 | 125 | return ssl.CertificateOptions(verify=False) | ||
862 | 126 | return super(LocalhostWhitelistedHTTPSPolicy, self).creatorForNetloc( | ||
863 | 127 | hostname, port) | ||
864 | 128 | |||
865 | 129 | |||
866 | 130 | class TestProberHTTPSProtocolAndFactory(TestCase): | ||
867 | 131 | layer = TwistedLayer | ||
868 | 132 | run_tests_with = AsynchronousDeferredRunTestForBrokenTwisted.make_factory( | ||
869 | 133 | timeout=30) | ||
870 | 134 | |||
871 | 135 | def setUp(self): | ||
872 | 136 | super(TestProberHTTPSProtocolAndFactory, self).setUp() | ||
873 | 137 | root = DistributionMirrorTestSecureHTTPServer() | ||
874 | 138 | site = server.Site(root) | ||
875 | 139 | site.displayTracebacks = False | ||
876 | 140 | keys_path = os.path.join(config.root, "configs", "development") | ||
877 | 141 | keys = ssl.DefaultOpenSSLContextFactory( | ||
878 | 142 | os.path.join(keys_path, "launchpad.key"), | ||
879 | 143 | os.path.join(keys_path, "launchpad.crt"), | ||
880 | 144 | ) | ||
881 | 145 | self.listening_port = reactor.listenSSL(0, site, keys) | ||
882 | 146 | |||
883 | 147 | self.addCleanup(self.listening_port.stopListening) | ||
884 | 148 | |||
885 | 149 | # Change the default policy to accept localhost self-signed | ||
886 | 150 | # certificates. | ||
887 | 151 | original_probefactory_policy = ProberFactory.https_agent_policy | ||
888 | 152 | original_redirect_policy = ( | ||
889 | 153 | RedirectAwareProberFactory.https_agent_policy) | ||
890 | 154 | ProberFactory.https_agent_policy = LocalhostWhitelistedHTTPSPolicy | ||
891 | 155 | RedirectAwareProberFactory.https_agent_policy = ( | ||
892 | 156 | LocalhostWhitelistedHTTPSPolicy) | ||
893 | 157 | |||
894 | 158 | for factory in (ProberFactory, RedirectAwareProberFactory): | ||
895 | 159 | self.useFixture(MockPatchObject( | ||
896 | 160 | factory, "https_agent_policy", | ||
897 | 161 | LocalhostWhitelistedHTTPSPolicy)) | ||
898 | 162 | |||
899 | 163 | self.port = self.listening_port.getHost().port | ||
900 | 164 | |||
901 | 165 | self.urls = {'timeout': u'https://localhost:%s/timeout' % self.port, | ||
902 | 166 | '200': u'https://localhost:%s/valid-mirror' % self.port, | ||
903 | 167 | '500': u'https://localhost:%s/error' % self.port, | ||
904 | 168 | '404': u'https://localhost:%s/invalid-mirror' % self.port} | ||
905 | 169 | self.pushConfig('launchpad', http_proxy=None) | ||
906 | 170 | |||
907 | 171 | self.useFixture(MockPatchObject( | ||
908 | 172 | distributionmirror_prober, "host_requests", {})) | ||
909 | 173 | self.useFixture(MockPatchObject( | ||
910 | 174 | distributionmirror_prober, "host_timeouts", {})) | ||
911 | 175 | self.useFixture(MockPatchObject( | ||
912 | 176 | distributionmirror_prober, "invalid_certificate_hosts", set())) | ||
913 | 177 | |||
914 | 178 | def _createProberAndProbe(self, url): | ||
915 | 179 | prober = ProberFactory(url) | ||
916 | 180 | return prober.probe() | ||
917 | 181 | |||
918 | 182 | def test_timeout(self): | ||
919 | 183 | prober = ProberFactory(self.urls['timeout'], timeout=0.5) | ||
920 | 184 | d = prober.probe() | ||
921 | 185 | return assert_fails_with(d, ProberTimeout) | ||
922 | 186 | |||
923 | 187 | def test_500(self): | ||
924 | 188 | d = self._createProberAndProbe(self.urls['500']) | ||
925 | 189 | return assert_fails_with(d, BadResponseCode) | ||
926 | 190 | |||
927 | 191 | def test_notfound(self): | ||
928 | 192 | d = self._createProberAndProbe(self.urls['404']) | ||
929 | 193 | return assert_fails_with(d, BadResponseCode) | ||
930 | 194 | |||
931 | 195 | def test_config_no_https_proxy(self): | ||
932 | 196 | prober = ProberFactory(self.urls['200']) | ||
933 | 197 | self.assertThat(prober, MatchesStructure.byEquality( | ||
934 | 198 | request_scheme='https', | ||
935 | 199 | request_host='localhost', | ||
936 | 200 | request_port=self.port, | ||
937 | 201 | request_path='/valid-mirror', | ||
938 | 202 | connect_scheme='https', | ||
939 | 203 | connect_host='localhost', | ||
940 | 204 | connect_port=self.port, | ||
941 | 205 | connect_path='/valid-mirror')) | ||
942 | 206 | |||
943 | 207 | def test_RedirectAwareProber_follows_https_redirect(self): | ||
944 | 208 | url = 'https://localhost:%s/redirect-to-valid-mirror/file' % self.port | ||
945 | 209 | prober = RedirectAwareProberFactory(url) | ||
946 | 210 | self.assertEqual(prober.url, url) | ||
947 | 211 | deferred = prober.probe() | ||
948 | 212 | |||
949 | 213 | def got_result(result): | ||
950 | 214 | self.assertEqual(http_client.OK, result.code) | ||
951 | 215 | self.assertEqual( | ||
952 | 216 | 'https://localhost:%s/valid-mirror/file' % self.port, | ||
953 | 217 | result.request.absoluteURI) | ||
954 | 218 | |||
955 | 219 | return deferred.addCallback(got_result) | ||
956 | 220 | |||
957 | 221 | def test_https_prober_uses_proxy(self): | ||
958 | 222 | root = DistributionMirrorTestSecureHTTPServer() | ||
959 | 223 | site = server.Site(root) | ||
960 | 224 | proxy_listen_port = reactor.listenTCP(0, site) | ||
961 | 225 | proxy_port = proxy_listen_port.getHost().port | ||
962 | 226 | self.pushConfig( | ||
963 | 227 | 'launchpad', http_proxy='http://localhost:%s/valid-mirror/file' | ||
964 | 228 | % proxy_port) | ||
965 | 229 | |||
966 | 230 | url = 'https://localhost:%s/valid-mirror/file' % self.port | ||
967 | 231 | prober = RedirectAwareProberFactory(url) | ||
968 | 232 | self.assertEqual(prober.url, url) | ||
969 | 233 | deferred = prober.probe() | ||
970 | 234 | |||
971 | 235 | def got_result(result): | ||
972 | 236 | # We basically don't care about the result here. We just want to | ||
973 | 237 | # check that it did the request to the correct URI, | ||
974 | 238 | # and ProxyAgent was used pointing to the correct proxy. | ||
975 | 239 | agent = prober.getHttpsClient()._agent | ||
976 | 240 | self.assertIsInstance(agent, ProxyAgent) | ||
977 | 241 | self.assertEqual('localhost', agent._proxyEndpoint._hostText) | ||
978 | 242 | self.assertEqual(proxy_port, agent._proxyEndpoint._port) | ||
979 | 243 | |||
980 | 244 | self.assertEqual( | ||
981 | 245 | 'https://localhost:%s/valid-mirror/file' % self.port, | ||
982 | 246 | result.value.response.request.absoluteURI) | ||
983 | 247 | |||
984 | 248 | def cleanup(*args, **kwargs): | ||
985 | 249 | proxy_listen_port.stopListening() | ||
986 | 250 | |||
987 | 251 | # Doing the proxy checks on the error callback because the | ||
988 | 252 | # proxy is dummy and always returns 404. | ||
989 | 253 | deferred.addErrback(got_result) | ||
990 | 254 | deferred.addBoth(cleanup) | ||
991 | 255 | return deferred | ||
992 | 256 | |||
993 | 257 | def test_https_fails_on_invalid_certificates(self): | ||
994 | 258 | """Changes set back the default browser-like policy for HTTPS | ||
995 | 259 | request and make sure the request is failing due to invalid | ||
996 | 260 | (self-signed) certificate. | ||
997 | 261 | """ | ||
998 | 262 | url = 'https://localhost:%s/valid-mirror/file' % self.port | ||
999 | 263 | prober = RedirectAwareProberFactory(url) | ||
1000 | 264 | prober.https_agent_policy = BrowserLikePolicyForHTTPS | ||
1001 | 265 | self.assertEqual(prober.url, url) | ||
1002 | 266 | deferred = prober.probe() | ||
1003 | 267 | |||
1004 | 268 | def on_failure(result): | ||
1005 | 269 | self.assertIsInstance(result.value, InvalidHTTPSCertificate) | ||
1006 | 270 | self.assertIn( | ||
1007 | 271 | ("localhost", self.port), | ||
1008 | 272 | distributionmirror_prober.invalid_certificate_hosts) | ||
1009 | 273 | |||
1010 | 274 | def on_success(result): | ||
1011 | 275 | if result is not None: | ||
1012 | 276 | self.fail( | ||
1013 | 277 | "Should have raised SSL error. Got '%s' instead" % result) | ||
1014 | 278 | |||
1015 | 279 | deferred.addErrback(on_failure) | ||
1016 | 280 | deferred.addCallback(on_success) | ||
1017 | 281 | return deferred | ||
1018 | 282 | |||
1019 | 283 | def test_https_skips_invalid_certificates_hosts(self): | ||
1020 | 284 | distributionmirror_prober.invalid_certificate_hosts.add( | ||
1021 | 285 | ("localhost", self.port)) | ||
1022 | 286 | url = 'https://localhost:%s/valid-mirror/file' % self.port | ||
1023 | 287 | prober = RedirectAwareProberFactory(url) | ||
1024 | 288 | prober.https_agent_policy = BrowserLikePolicyForHTTPS | ||
1025 | 289 | self.assertEqual(prober.url, url) | ||
1026 | 290 | deferred = prober.probe() | ||
1027 | 291 | |||
1028 | 292 | return assert_fails_with(deferred, InvalidHTTPSCertificateSkipped) | ||
1029 | 293 | |||
1030 | 294 | |||
1031 | 295 | class TestProberProtocolAndFactory(TestCase): | 106 | class TestProberProtocolAndFactory(TestCase): |
1032 | 296 | 107 | ||
1033 | 297 | layer = TwistedLayer | 108 | layer = TwistedLayer |
1034 | @@ -401,7 +212,7 @@ class TestProberProtocolAndFactory(TestCase): | |||
1035 | 401 | self.assertTrue(prober.url == new_url) | 212 | self.assertTrue(prober.url == new_url) |
1036 | 402 | self.assertTrue(result == str(http_client.OK)) | 213 | self.assertTrue(result == str(http_client.OK)) |
1037 | 403 | 214 | ||
1039 | 404 | return deferred.addBoth(got_result) | 215 | return deferred.addCallback(got_result) |
1040 | 405 | 216 | ||
1041 | 406 | def test_redirectawareprober_detects_infinite_loop(self): | 217 | def test_redirectawareprober_detects_infinite_loop(self): |
1042 | 407 | prober = RedirectAwareProberFactory( | 218 | prober = RedirectAwareProberFactory( |
1043 | @@ -926,16 +737,12 @@ class TestMirrorCDImageProberCallbacks(TestCaseWithFactory): | |||
1044 | 926 | ConnectionSkipped, | 737 | ConnectionSkipped, |
1045 | 927 | RedirectToDifferentFile, | 738 | RedirectToDifferentFile, |
1046 | 928 | UnknownURLSchemeAfterRedirect, | 739 | UnknownURLSchemeAfterRedirect, |
1047 | 929 | InvalidHTTPSCertificate, | ||
1048 | 930 | InvalidHTTPSCertificateSkipped, | ||
1049 | 931 | ])) | 740 | ])) |
1050 | 932 | exceptions = [BadResponseCode(str(http_client.NOT_FOUND)), | 741 | exceptions = [BadResponseCode(str(http_client.NOT_FOUND)), |
1051 | 933 | ProberTimeout('http://localhost/', 5), | 742 | ProberTimeout('http://localhost/', 5), |
1052 | 934 | ConnectionSkipped(), | 743 | ConnectionSkipped(), |
1053 | 935 | RedirectToDifferentFile('/foo', '/bar'), | 744 | RedirectToDifferentFile('/foo', '/bar'), |
1057 | 936 | UnknownURLSchemeAfterRedirect('https://localhost'), | 745 | UnknownURLSchemeAfterRedirect('https://localhost')] |
1055 | 937 | InvalidHTTPSCertificate('localhost', 443), | ||
1056 | 938 | InvalidHTTPSCertificateSkipped("https://localhost/xx")] | ||
1058 | 939 | for exception in exceptions: | 746 | for exception in exceptions: |
1059 | 940 | failure = callbacks.ensureOrDeleteMirrorCDImageSeries( | 747 | failure = callbacks.ensureOrDeleteMirrorCDImageSeries( |
1060 | 941 | [(defer.FAILURE, Failure(exception))]) | 748 | [(defer.FAILURE, Failure(exception))]) |
1061 | diff --git a/lib/lp/scripts/utilities/importpedant.py b/lib/lp/scripts/utilities/importpedant.py | |||
1062 | index c7859a4..ec41baf 100644 | |||
1063 | --- a/lib/lp/scripts/utilities/importpedant.py | |||
1064 | +++ b/lib/lp/scripts/utilities/importpedant.py | |||
1065 | @@ -1,4 +1,4 @@ | |||
1067 | 1 | # Copyright 2009-2020 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2016 Canonical Ltd. This software is licensed under the |
1068 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
1069 | 3 | 3 | ||
1070 | 4 | from __future__ import absolute_import, print_function, unicode_literals | 4 | from __future__ import absolute_import, print_function, unicode_literals |
1071 | @@ -42,8 +42,6 @@ valid_imports_not_in_all = { | |||
1072 | 42 | 'textwrap': set(['dedent']), | 42 | 'textwrap': set(['dedent']), |
1073 | 43 | 'testtools.testresult.real': set(['_details_to_str']), | 43 | 'testtools.testresult.real': set(['_details_to_str']), |
1074 | 44 | 'twisted.internet.threads': set(['deferToThreadPool']), | 44 | 'twisted.internet.threads': set(['deferToThreadPool']), |
1075 | 45 | # Even docs tell us to use this class. See docs on WebClientContextFactory. | ||
1076 | 46 | 'twisted.web.client': set(['BrowserLikePolicyForHTTPS']), | ||
1077 | 47 | 'zope.component': set( | 45 | 'zope.component': set( |
1078 | 48 | ['adapter', | 46 | ['adapter', |
1079 | 49 | 'ComponentLookupError', | 47 | 'ComponentLookupError', |
1080 | diff --git a/lib/lp/testing/factory.py b/lib/lp/testing/factory.py | |||
1081 | index 6c0492f..85a05b6 100644 | |||
1082 | --- a/lib/lp/testing/factory.py | |||
1083 | +++ b/lib/lp/testing/factory.py | |||
1084 | @@ -2,7 +2,7 @@ | |||
1085 | 2 | # NOTE: The first line above must stay first; do not move the copyright | 2 | # NOTE: The first line above must stay first; do not move the copyright |
1086 | 3 | # notice to the top. See http://www.python.org/dev/peps/pep-0263/. | 3 | # notice to the top. See http://www.python.org/dev/peps/pep-0263/. |
1087 | 4 | # | 4 | # |
1089 | 5 | # Copyright 2009-2020 Canonical Ltd. This software is licensed under the | 5 | # Copyright 2009-2019 Canonical Ltd. This software is licensed under the |
1090 | 6 | # GNU Affero General Public License version 3 (see the file LICENSE). | 6 | # GNU Affero General Public License version 3 (see the file LICENSE). |
1091 | 7 | 7 | ||
1092 | 8 | """Testing infrastructure for the Launchpad application. | 8 | """Testing infrastructure for the Launchpad application. |
1093 | @@ -3527,13 +3527,13 @@ class BareLaunchpadObjectFactory(ObjectFactory): | |||
1094 | 3527 | return proberecord | 3527 | return proberecord |
1095 | 3528 | 3528 | ||
1096 | 3529 | def makeMirror(self, distribution, displayname=None, country=None, | 3529 | def makeMirror(self, distribution, displayname=None, country=None, |
1098 | 3530 | http_url=None, https_url=None, ftp_url=None, rsync_url=None, | 3530 | http_url=None, ftp_url=None, rsync_url=None, |
1099 | 3531 | official_candidate=False): | 3531 | official_candidate=False): |
1100 | 3532 | """Create a mirror for the distribution.""" | 3532 | """Create a mirror for the distribution.""" |
1101 | 3533 | if displayname is None: | 3533 | if displayname is None: |
1102 | 3534 | displayname = self.getUniqueString("mirror") | 3534 | displayname = self.getUniqueString("mirror") |
1103 | 3535 | # If no URL is specified create an HTTP URL. | 3535 | # If no URL is specified create an HTTP URL. |
1105 | 3536 | if http_url is https_url is ftp_url is rsync_url is None: | 3536 | if http_url is None and ftp_url is None and rsync_url is None: |
1106 | 3537 | http_url = self.getUniqueURL() | 3537 | http_url = self.getUniqueURL() |
1107 | 3538 | # If no country is given use Argentina. | 3538 | # If no country is given use Argentina. |
1108 | 3539 | if country is None: | 3539 | if country is None: |
1109 | @@ -3547,7 +3547,6 @@ class BareLaunchpadObjectFactory(ObjectFactory): | |||
1110 | 3547 | display_name=displayname, | 3547 | display_name=displayname, |
1111 | 3548 | description=None, | 3548 | description=None, |
1112 | 3549 | http_base_url=http_url, | 3549 | http_base_url=http_url, |
1113 | 3550 | https_base_url=https_url, | ||
1114 | 3551 | ftp_base_url=ftp_url, | 3550 | ftp_base_url=ftp_url, |
1115 | 3552 | rsync_base_url=rsync_url, | 3551 | rsync_base_url=rsync_url, |
1116 | 3553 | official_candidate=official_candidate) | 3552 | official_candidate=official_candidate) |
Voting criteria not met /jenkins. ols.canonical. com/online- services/ job/launchpad/ 212/
https:/