Merge ~pappacena/launchpad:https-mirrors-2 into launchpad:master

Proposed by Thiago F. Pappacena
Status: Merged
Approved by: Thiago F. Pappacena
Approved revision: 7448ccd84ef29a536bac827ece5b8116ae73664a
Merge reported by: Otto Co-Pilot
Merged at revision: not available
Proposed branch: ~pappacena/launchpad:https-mirrors-2
Merge into: launchpad:master
Diff against target: 1116 lines (+477/-53)
16 files modified
lib/lp/registry/browser/distributionmirror.py (+7/-6)
lib/lp/registry/browser/tests/distributionmirror-views.txt (+40/-9)
lib/lp/registry/configure.zcml (+5/-3)
lib/lp/registry/interfaces/distribution.py (+4/-4)
lib/lp/registry/interfaces/distributionmirror.py (+21/-3)
lib/lp/registry/model/distribution.py (+7/-4)
lib/lp/registry/model/distributionmirror.py (+10/-2)
lib/lp/registry/scripts/distributionmirror_prober.py (+153/-7)
lib/lp/registry/stories/webservice/xx-distribution-mirror.txt (+4/-0)
lib/lp/registry/stories/webservice/xx-distribution.txt (+1/-0)
lib/lp/registry/templates/distributionmirror-index.pt (+4/-0)
lib/lp/registry/templates/distributionmirror-macros.pt (+3/-1)
lib/lp/registry/tests/distributionmirror_http_server.py (+15/-7)
lib/lp/registry/tests/test_distributionmirror_prober.py (+196/-3)
lib/lp/scripts/utilities/importpedant.py (+3/-1)
lib/lp/testing/factory.py (+4/-3)
Reviewer Review Type Date Requested Status
Colin Watson (community) Approve
Review via email: mp+379913@code.launchpad.net

Commit message

Undoing git revert on: Allowing to register HTTPS mirrors (both for CD images and archives)

Description of the change

This MP is undoing the git revert of HTTPS mirrors feature on master (https://code.launchpad.net/~pappacena/launchpad/+git/launchpad/+merge/379387). This feature got reverted because the database patch was not merged yet, so the code should wait for its deployment to production before being landed.

To post a comment you must log in.
Revision history for this message
Colin Watson (cjwatson) wrote :

Thanks. You can go ahead and re-land this now; the database patch has been deployed to production, and I've merged that revision of db-devel into master.

review: Approve
Revision history for this message
Thiago F. Pappacena (pappacena) wrote :

Great! Thanks, cjwatson!

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/lib/lp/registry/browser/distributionmirror.py b/lib/lp/registry/browser/distributionmirror.py
2index fd671a0..6ae1eb7 100644
3--- a/lib/lp/registry/browser/distributionmirror.py
4+++ b/lib/lp/registry/browser/distributionmirror.py
5@@ -1,4 +1,4 @@
6-# Copyright 2009 Canonical Ltd. This software is licensed under the
7+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
8 # GNU Affero General Public License version 3 (see the file LICENSE).
9
10 __metaclass__ = type
11@@ -207,9 +207,9 @@ class DistributionMirrorDeleteView(LaunchpadFormView):
12 class DistributionMirrorAddView(LaunchpadFormView):
13 schema = IDistributionMirror
14 field_names = [
15- "display_name", "description", "whiteboard", "http_base_url",
16- "ftp_base_url", "rsync_base_url", "speed", "country", "content",
17- "official_candidate",
18+ "display_name", "description", "whiteboard", "https_base_url",
19+ "http_base_url", "ftp_base_url", "rsync_base_url", "speed", "country",
20+ "content", "official_candidate",
21 ]
22 invariant_context = None
23
24@@ -235,6 +235,7 @@ class DistributionMirrorAddView(LaunchpadFormView):
25 content=data['content'], display_name=data['display_name'],
26 description=data['description'],
27 whiteboard=data['whiteboard'],
28+ https_base_url=data['https_base_url'],
29 http_base_url=data['http_base_url'],
30 ftp_base_url=data['ftp_base_url'],
31 rsync_base_url=data['rsync_base_url'],
32@@ -279,8 +280,8 @@ class DistributionMirrorEditView(LaunchpadEditFormView):
33 schema = IDistributionMirror
34 field_names = [
35 "name", "display_name", "description", "whiteboard",
36- "http_base_url", "ftp_base_url", "rsync_base_url", "speed",
37- "country", "content", "official_candidate",
38+ "https_base_url", "http_base_url", "ftp_base_url", "rsync_base_url",
39+ "speed", "country", "content", "official_candidate",
40 ]
41
42 @property
43diff --git a/lib/lp/registry/browser/tests/distributionmirror-views.txt b/lib/lp/registry/browser/tests/distributionmirror-views.txt
44index e11dbd9..f8fe4c5 100644
45--- a/lib/lp/registry/browser/tests/distributionmirror-views.txt
46+++ b/lib/lp/registry/browser/tests/distributionmirror-views.txt
47@@ -44,18 +44,19 @@ The view provides a label, page_title, and cancel_url
48 >>> print view.cancel_url
49 http://launchpad.test/ubuntu
50
51-A HTTP or FTP URL is required to register a mirror.
52+A HTTP, HTTPS or FTP URL is required to register a mirror.
53
54 >>> view.field_names
55- ['display_name', 'description', 'whiteboard', 'http_base_url',
56- 'ftp_base_url', 'rsync_base_url', 'speed', 'country', 'content',
57- 'official_candidate']
58+ ['display_name', 'description', 'whiteboard', 'https_base_url',
59+ 'http_base_url', 'ftp_base_url', 'rsync_base_url', 'speed', 'country',
60+ 'content', 'official_candidate']
61
62 >>> form = {
63 ... 'field.display_name': 'Illuminati',
64 ... 'field.description': 'description',
65 ... 'field.whiteboard': 'whiteboard',
66 ... 'field.http_base_url': 'http://secret.me/',
67+ ... 'field.https_base_url': '',
68 ... 'field.ftp_base_url': '',
69 ... 'field.rsync_base_url': '',
70 ... 'field.speed': 'S128K',
71@@ -93,6 +94,7 @@ not significant).
72 The same is true for a FTP URL.
73
74 >>> mirror.ftp_base_url = 'ftp://now-here.me/'
75+ >>> bad_form['field.https_base_url'] = ''
76 >>> bad_form['field.http_base_url'] = ''
77 >>> bad_form['field.ftp_base_url'] = 'ftp://now-here.me'
78 >>> view = create_initialized_view(ubuntu, '+newmirror', form=bad_form)
79@@ -110,14 +112,15 @@ The same is true for a rsync URL.
80 ... print error[2]
81 The distribution mirror ... is already registered with this URL.
82
83-A mirror must have an ftp or http URL.
84+A mirror must have an ftp, HTTPS or http URL.
85
86+ >>> bad_form['field.https_base_url'] = ''
87 >>> bad_form['field.http_base_url'] = ''
88 >>> bad_form['field.ftp_base_url'] = ''
89 >>> view = create_initialized_view(ubuntu, '+newmirror', form=bad_form)
90 >>> for message in view.errors:
91 ... print message
92- A mirror must have at least an HTTP or FTP URL.
93+ A mirror must have at least an HTTP(S) or FTP URL.
94
95 The URL cannot contain a fragment.
96
97@@ -135,6 +138,34 @@ The URL cannot contain a query string.
98 ... print error[2]
99 URIs with query strings are not allowed.
100
101+The HTTPS URL may not have an HTTP scheme.
102+
103+ >>> bad_form['field.http_base_url'] = ''
104+ >>> bad_form['field.https_base_url'] = 'http://secret.me/#fragement'
105+ >>> view = create_initialized_view(ubuntu, '+newmirror', form=bad_form)
106+ >>> for error in view.errors:
107+ ... print error[2]
108+ The URI scheme "http" is not allowed.
109+ Only URIs with the following schemes may be used: https
110+
111+The HTTPS URL cannot contain a fragment.
112+
113+ >>> bad_form['field.http_base_url'] = ''
114+ >>> bad_form['field.https_base_url'] = 'https://secret.me/#fragement'
115+ >>> view = create_initialized_view(ubuntu, '+newmirror', form=bad_form)
116+ >>> for error in view.errors:
117+ ... print error[2]
118+ URIs with fragment identifiers are not allowed.
119+
120+The URL cannot contain a query string.
121+
122+ >>> bad_form['field.http_base_url'] = ''
123+ >>> bad_form['field.https_base_url'] = 'https://secret.me/?query=string'
124+ >>> view = create_initialized_view(ubuntu, '+newmirror', form=bad_form)
125+ >>> for error in view.errors:
126+ ... print error[2]
127+ URIs with query strings are not allowed.
128+
129
130 Reviewing a distribution mirror
131 -------------------------------
132@@ -239,9 +270,9 @@ The +edit view provides a label, page_title, and cancel_url.
133 The user can edit the mirror fields.
134
135 >>> view.field_names
136- ['name', 'display_name', 'description', 'whiteboard', 'http_base_url',
137- 'ftp_base_url', 'rsync_base_url', 'speed', 'country', 'content',
138- 'official_candidate']
139+ ['name', 'display_name', 'description', 'whiteboard', 'https_base_url',
140+ 'http_base_url', 'ftp_base_url', 'rsync_base_url', 'speed', 'country',
141+ 'content', 'official_candidate']
142
143 >>> print mirror.ftp_base_url
144 None
145diff --git a/lib/lp/registry/configure.zcml b/lib/lp/registry/configure.zcml
146index 9be68e0..d5875c5 100644
147--- a/lib/lp/registry/configure.zcml
148+++ b/lib/lp/registry/configure.zcml
149@@ -1,4 +1,4 @@
150-<!-- Copyright 2009-2019 Canonical Ltd. This software is licensed under the
151+<!-- Copyright 2009-2020 Canonical Ltd. This software is licensed under the
152 GNU Affero General Public License version 3 (see the file LICENSE).
153 -->
154
155@@ -2046,6 +2046,7 @@
156 description
157 distribution
158 http_base_url
159+ https_base_url
160 ftp_base_url
161 rsync_base_url
162 enabled
163@@ -2084,8 +2085,9 @@
164 <require
165 permission="launchpad.Edit"
166 set_attributes="name display_name description whiteboard
167- http_base_url ftp_base_url rsync_base_url enabled
168- speed country content official_candidate owner"
169+ http_base_url https_base_url ftp_base_url
170+ rsync_base_url enabled speed country content
171+ official_candidate owner"
172 attributes="official_candidate whiteboard resubmitForReview" />
173 <require
174 permission="launchpad.Admin"
175diff --git a/lib/lp/registry/interfaces/distribution.py b/lib/lp/registry/interfaces/distribution.py
176index c7ddb95..a7106f6 100644
177--- a/lib/lp/registry/interfaces/distribution.py
178+++ b/lib/lp/registry/interfaces/distribution.py
179@@ -1,4 +1,4 @@
180-# Copyright 2009-2015 Canonical Ltd. This software is licensed under the
181+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
182 # GNU Affero General Public License version 3 (see the file LICENSE).
183
184 """Interfaces including and related to IDistribution."""
185@@ -491,13 +491,13 @@ class IDistributionPublic(
186 """Return the country DNS mirror for a country and content type."""
187
188 def newMirror(owner, speed, country, content, display_name=None,
189- description=None, http_base_url=None,
190+ description=None, http_base_url=None, https_base_url=None,
191 ftp_base_url=None, rsync_base_url=None, enabled=False,
192 official_candidate=False, whiteboard=None):
193 """Create a new DistributionMirror for this distribution.
194
195- At least one of http_base_url or ftp_base_url must be provided in
196- order to create a mirror.
197+ At least one of {http,https,ftp}_base_url must be provided in order to
198+ create a mirror.
199 """
200
201 def getOCIProject(name):
202diff --git a/lib/lp/registry/interfaces/distributionmirror.py b/lib/lp/registry/interfaces/distributionmirror.py
203index ce6720e..4849e7a 100644
204--- a/lib/lp/registry/interfaces/distributionmirror.py
205+++ b/lib/lp/registry/interfaces/distributionmirror.py
206@@ -1,4 +1,4 @@
207-# Copyright 2009 Canonical Ltd. This software is licensed under the
208+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
209 # GNU Affero General Public License version 3 (see the file LICENSE).
210
211 __metaclass__ = type
212@@ -302,6 +302,12 @@ class DistroMirrorHTTPURIField(DistroMirrorURIField):
213 return getUtility(IDistributionMirrorSet).getByHttpUrl(url)
214
215
216+class DistroMirrorHTTPSURIField(DistroMirrorURIField):
217+
218+ def getMirrorByURI(self, url):
219+ return getUtility(IDistributionMirrorSet).getByHttpsUrl(url)
220+
221+
222 class DistroMirrorFTPURIField(DistroMirrorURIField):
223
224 def getMirrorByURI(self, url):
225@@ -349,6 +355,14 @@ class IDistributionMirror(Interface):
226 allowed_schemes=['http'], allow_userinfo=False,
227 allow_query=False, allow_fragment=False, trailing_slash=True,
228 description=_('e.g.: http://archive.ubuntu.com/ubuntu/')))
229+ https_base_url = exported(DistroMirrorHTTPSURIField(
230+ title=_('HTTPS URL'), required=False, readonly=False,
231+ allowed_schemes=['https'], allow_userinfo=False,
232+ allow_query=False, allow_fragment=False, trailing_slash=True,
233+ # XXX: pappacena 2020-02-21: Add description field with a more
234+ # suitable example once we have https for archive.ubuntu.com, like:
235+ # description=_('e.g.: http://archive.ubuntu.com/ubuntu/')
236+ ))
237 ftp_base_url = exported(DistroMirrorFTPURIField(
238 title=_('FTP URL'), required=False, readonly=False,
239 allowed_schemes=['ftp'], allow_userinfo=False,
240@@ -435,8 +449,9 @@ class IDistributionMirror(Interface):
241
242 @invariant
243 def mirrorMustHaveHTTPOrFTPURL(mirror):
244- if not (mirror.http_base_url or mirror.ftp_base_url):
245- raise Invalid('A mirror must have at least an HTTP or FTP URL.')
246+ if not (mirror.http_base_url or mirror.https_base_url or
247+ mirror.ftp_base_url):
248+ raise Invalid('A mirror must have at least an HTTP(S) or FTP URL.')
249
250 def getSummarizedMirroredSourceSeries():
251 """Return a summarized list of this distribution_mirror's
252@@ -614,6 +629,9 @@ class IDistributionMirrorSet(Interface):
253 def getByHttpUrl(url):
254 """Return the mirror with the given HTTP URL or None."""
255
256+ def getByHttpsUrl(url):
257+ """Return the mirror with the given HTTPS URL or None."""
258+
259 def getByFtpUrl(url):
260 """Return the mirror with the given FTP URL or None."""
261
262diff --git a/lib/lp/registry/model/distribution.py b/lib/lp/registry/model/distribution.py
263index 2288d0e..f78908b 100644
264--- a/lib/lp/registry/model/distribution.py
265+++ b/lib/lp/registry/model/distribution.py
266@@ -1,4 +1,4 @@
267-# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
268+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
269 # GNU Affero General Public License version 3 (see the file LICENSE).
270
271 """Database classes for implementing distribution items."""
272@@ -709,7 +709,7 @@ class Distribution(SQLBase, BugTargetBase, MakesAnnouncements,
273 country_dns_mirror=True).one()
274
275 def newMirror(self, owner, speed, country, content, display_name=None,
276- description=None, http_base_url=None,
277+ description=None, http_base_url=None, https_base_url=None,
278 ftp_base_url=None, rsync_base_url=None,
279 official_candidate=False, enabled=False,
280 whiteboard=None):
281@@ -722,15 +722,17 @@ class Distribution(SQLBase, BugTargetBase, MakesAnnouncements,
282 return None
283
284 urls = {'http_base_url': http_base_url,
285+ 'https_base_url': https_base_url,
286 'ftp_base_url': ftp_base_url,
287 'rsync_base_url': rsync_base_url}
288 for name, value in urls.items():
289 if value is not None:
290 urls[name] = IDistributionMirror[name].normalize(value)
291
292- url = urls['http_base_url'] or urls['ftp_base_url']
293+ url = (urls['https_base_url'] or urls['http_base_url'] or
294+ urls['ftp_base_url'])
295 assert url is not None, (
296- "A mirror must provide either an HTTP or FTP URL (or both).")
297+ "A mirror must provide at least one HTTP/HTTPS/FTP URL.")
298 dummy, host, dummy, dummy, dummy, dummy = urlparse(url)
299 name = sanitize_name('%s-%s' % (host, content.name.lower()))
300
301@@ -744,6 +746,7 @@ class Distribution(SQLBase, BugTargetBase, MakesAnnouncements,
302 distribution=self, owner=owner, name=name, speed=speed,
303 country=country, content=content, display_name=display_name,
304 description=description, http_base_url=urls['http_base_url'],
305+ https_base_url=urls['https_base_url'],
306 ftp_base_url=urls['ftp_base_url'],
307 rsync_base_url=urls['rsync_base_url'],
308 official_candidate=official_candidate, enabled=enabled,
309diff --git a/lib/lp/registry/model/distributionmirror.py b/lib/lp/registry/model/distributionmirror.py
310index 747a2b0..323fb26 100644
311--- a/lib/lp/registry/model/distributionmirror.py
312+++ b/lib/lp/registry/model/distributionmirror.py
313@@ -1,4 +1,4 @@
314-# Copyright 2009-2015 Canonical Ltd. This software is licensed under the
315+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
316 # GNU Affero General Public License version 3 (see the file LICENSE).
317
318 """Module docstring goes here."""
319@@ -129,6 +129,8 @@ class DistributionMirror(SQLBase):
320 notNull=False, default=None)
321 http_base_url = StringCol(
322 notNull=False, default=None, unique=True)
323+ https_base_url = StringCol(
324+ notNull=False, default=None, unique=True)
325 ftp_base_url = StringCol(
326 notNull=False, default=None, unique=True)
327 rsync_base_url = StringCol(
328@@ -155,7 +157,9 @@ class DistributionMirror(SQLBase):
329 @property
330 def base_url(self):
331 """See IDistributionMirror"""
332- if self.http_base_url is not None:
333+ if self.https_base_url is not None:
334+ return self.https_base_url
335+ elif self.http_base_url is not None:
336 return self.http_base_url
337 else:
338 return self.ftp_base_url
339@@ -677,6 +681,10 @@ class DistributionMirrorSet:
340 """See IDistributionMirrorSet"""
341 return DistributionMirror.selectOneBy(http_base_url=url)
342
343+ def getByHttpsUrl(self, url):
344+ """See IDistributionMirrorSet"""
345+ return DistributionMirror.selectOneBy(https_base_url=url)
346+
347 def getByFtpUrl(self, url):
348 """See IDistributionMirrorSet"""
349 return DistributionMirror.selectOneBy(ftp_base_url=url)
350diff --git a/lib/lp/registry/scripts/distributionmirror_prober.py b/lib/lp/registry/scripts/distributionmirror_prober.py
351index 916d4ba..c633fb8 100644
352--- a/lib/lp/registry/scripts/distributionmirror_prober.py
353+++ b/lib/lp/registry/scripts/distributionmirror_prober.py
354@@ -1,4 +1,4 @@
355-# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
356+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
357 # GNU Affero General Public License version 3 (see the file LICENSE).
358
359 __metaclass__ = type
360@@ -12,6 +12,7 @@ import logging
361 import os.path
362 from StringIO import StringIO
363
364+import OpenSSL
365 import requests
366 from six.moves import http_client
367 from six.moves.urllib.parse import (
368@@ -20,14 +21,27 @@ from six.moves.urllib.parse import (
369 urlparse,
370 urlunparse,
371 )
372+from treq.client import HTTPClient as TreqHTTPClient
373 from twisted.internet import (
374 defer,
375 protocol,
376 reactor,
377 )
378-from twisted.internet.defer import DeferredSemaphore
379+from twisted.internet.defer import (
380+ CancelledError,
381+ DeferredSemaphore,
382+ )
383+from twisted.internet.endpoints import HostnameEndpoint
384+from twisted.internet.ssl import VerificationError
385 from twisted.python.failure import Failure
386+from twisted.web.client import (
387+ Agent,
388+ BrowserLikePolicyForHTTPS,
389+ ProxyAgent,
390+ ResponseNeverReceived,
391+ )
392 from twisted.web.http import HTTPClient
393+from twisted.web.iweb import IResponse
394 from zope.component import getUtility
395
396 from lp.app.interfaces.launchpad import ILaunchpadCelebrities
397@@ -50,6 +64,7 @@ from lp.soyuz.interfaces.distroarchseries import IDistroArchSeries
398 # IMPORTANT: Changing these values can cause lots of false negatives when
399 # probing mirrors, so please don't change them unless you know what you're
400 # doing.
401+
402 MIN_REQUEST_TIMEOUT_RATIO = 3
403 MIN_REQUESTS_TO_CONSIDER_RATIO = 30
404
405@@ -57,6 +72,9 @@ MIN_REQUESTS_TO_CONSIDER_RATIO = 30
406 # We need to get rid of these global dicts in this module.
407 host_requests = {}
408 host_timeouts = {}
409+# Set of invalid certificate (host, port) tuples, to avoid doing HTTPS calls
410+# to hosts we already know they are not valid.
411+invalid_certificate_hosts = set()
412
413 MAX_REDIRECTS = 3
414
415@@ -161,6 +179,64 @@ class ProberProtocol(HTTPClient):
416 pass
417
418
419+class HTTPSProbeFailureHandler:
420+ """Handler to translate general errors into expected errors on HTTPS
421+ connections."""
422+ def __init__(self, factory):
423+ self.factory = factory
424+
425+ def handleResponse(self, response):
426+ """Translates any request with return code different from 200 into
427+ an error in the callback chain.
428+
429+ Note that other 2xx codes that are not 200 are considered errors too.
430+ This behaviour is the same as seen in ProberProtocol.handleStatus,
431+ for HTTP responses.
432+ """
433+ status = response.code
434+ if status == http_client.OK:
435+ return response
436+ else:
437+ raise BadResponseCode(status, response)
438+
439+ def handleErrors(self, error):
440+ """Handle exceptions in https requests.
441+ """
442+ if self.isInvalidCertificateError(error):
443+ invalid_certificate_hosts.add(
444+ (self.factory.request_host, self.factory.request_port))
445+ reason = InvalidHTTPSCertificate(
446+ self.factory.request_host, self.factory.request_port)
447+ raise reason
448+ if self.isTimeout(error):
449+ raise ProberTimeout(self.factory.url, self.factory.timeout)
450+ raise error
451+
452+ def isTimeout(self, error):
453+ """Checks if the error was caused by a timeout.
454+ """
455+ return self._isErrorFromType(error, CancelledError)
456+
457+ def isInvalidCertificateError(self, error):
458+ """Checks if the error was caused by an invalid certificate.
459+ """
460+ # It might be a raw SSL error, or a twisted-encapsulated
461+ # verification error (such as DNSMismatch error when the
462+ # certificate is valid for a different domain, for example).
463+ return self._isErrorFromType(
464+ error, OpenSSL.SSL.Error, VerificationError)
465+
466+ def _isErrorFromType(self, error, *types):
467+ """Checks if the error was caused by any of the given types.
468+ """
469+ if not isinstance(error.value, ResponseNeverReceived):
470+ return False
471+ for reason in error.value.reasons:
472+ if reason.check(*types) is not None:
473+ return True
474+ return False
475+
476+
477 class RedirectAwareProberProtocol(ProberProtocol):
478 """A specialized version of ProberProtocol that follows HTTP redirects."""
479
480@@ -216,6 +292,8 @@ class ProberFactory(protocol.ClientFactory):
481 connect_port = None
482 connect_path = None
483
484+ https_agent_policy = BrowserLikePolicyForHTTPS
485+
486 def __init__(self, url, timeout=config.distributionmirrorprober.timeout):
487 # We want the deferred to be a private attribute (_deferred) to make
488 # sure our clients will only use the deferred returned by the probe()
489@@ -225,9 +303,14 @@ class ProberFactory(protocol.ClientFactory):
490 self.timeout = timeout
491 self.timeoutCall = None
492 self.setURL(url.encode('ascii'))
493+ self.logger = logging.getLogger('distributionmirror-prober')
494+
495+ @property
496+ def is_https(self):
497+ return self.request_scheme == 'https'
498
499 def probe(self):
500- logger = logging.getLogger('distributionmirror-prober')
501+ logger = self.logger
502 # NOTE: We don't want to issue connections to any outside host when
503 # running the mirror prober in a development machine, so we do this
504 # hack here.
505@@ -244,13 +327,46 @@ class ProberFactory(protocol.ClientFactory):
506 "host already." % self.url)
507 return self._deferred
508
509+ if (self.request_host, self.request_port) in invalid_certificate_hosts:
510+ reactor.callLater(
511+ 0, self.failed, InvalidHTTPSCertificateSkipped(self.url))
512+ logger.debug("Skipping %s as it doesn't have a valid HTTPS "
513+ "certificate" % self.url)
514+ return self._deferred
515+
516 self.connect()
517 logger.debug('Probing %s' % self.url)
518 return self._deferred
519
520+ def getHttpsClient(self):
521+ # Should we use a proxy?
522+ if not config.launchpad.http_proxy:
523+ agent = Agent(
524+ reactor=reactor, contextFactory=self.https_agent_policy())
525+ else:
526+ endpoint = HostnameEndpoint(
527+ reactor, self.connect_host, self.connect_port)
528+ agent = ProxyAgent(endpoint)
529+ return TreqHTTPClient(agent)
530+
531 def connect(self):
532+ """Starts the connection and sets the self._deferred to the proper
533+ task.
534+ """
535 host_requests[self.request_host] += 1
536- reactor.connectTCP(self.connect_host, self.connect_port, self)
537+ if self.is_https:
538+ treq = self.getHttpsClient()
539+ self._deferred.addCallback(
540+ lambda _: treq.head(
541+ self.url, reactor=reactor, allow_redirects=True,
542+ timeout=self.timeout))
543+ error_handler = HTTPSProbeFailureHandler(self)
544+ self._deferred.addCallback(error_handler.handleResponse)
545+ self._deferred.addErrback(error_handler.handleErrors)
546+ reactor.callWhenRunning(self._deferred.callback, None)
547+ else:
548+ reactor.connectTCP(self.connect_host, self.connect_port, self)
549+
550 if self.timeoutCall is not None and self.timeoutCall.active():
551 self._cancelTimeout(None)
552 self.timeoutCall = reactor.callLater(
553@@ -269,6 +385,8 @@ class ProberFactory(protocol.ClientFactory):
554 self.connector = connector
555
556 def succeeded(self, status):
557+ if IResponse.providedBy(status):
558+ status = str(status.code)
559 self._deferred.callback(status)
560
561 def failed(self, reason):
562@@ -288,7 +406,7 @@ class ProberFactory(protocol.ClientFactory):
563 # https://bugs.squid-cache.org/show_bug.cgi?id=1758 applied. So, if
564 # you encounter any problems with FTP URLs you'll probably have to nag
565 # the sysadmins to fix squid for you.
566- if scheme not in ('http', 'ftp'):
567+ if scheme not in ('http', 'https', 'ftp'):
568 raise UnknownURLScheme(url)
569
570 if scheme and host:
571@@ -376,14 +494,26 @@ class ProberTimeout(ProberError):
572
573 class BadResponseCode(ProberError):
574
575- def __init__(self, status, *args):
576+ def __init__(self, status, response=None, *args):
577 ProberError.__init__(self, *args)
578 self.status = status
579+ self.response = response
580
581 def __str__(self):
582 return "Bad response code: %s" % self.status
583
584
585+class InvalidHTTPSCertificate(ProberError):
586+ def __init__(self, host, port, *args):
587+ super(InvalidHTTPSCertificate, self).__init__(*args)
588+ self.host = host
589+ self.port = port
590+
591+ def __str__(self):
592+ return "Invalid SSL certificate when trying to probe %s:%s" % (
593+ self.host, self.port)
594+
595+
596 class RedirectToDifferentFile(ProberError):
597
598 def __init__(self, orig_path, new_path, *args):
599@@ -409,6 +539,14 @@ class ConnectionSkipped(ProberError):
600 "host. It will be retried on the next probing run.")
601
602
603+class InvalidHTTPSCertificateSkipped(ProberError):
604+
605+ def __str__(self):
606+ return ("Connection skipped because the server doesn't have a valid "
607+ "HTTPS certificate. It will be retried on the next "
608+ "probing run.")
609+
610+
611 class UnknownURLScheme(ProberError):
612
613 def __init__(self, url, *args):
614@@ -429,7 +567,13 @@ class UnknownURLSchemeAfterRedirect(UnknownURLScheme):
615
616 class ArchiveMirrorProberCallbacks(LoggingMixin):
617
618- expected_failures = (BadResponseCode, ProberTimeout, ConnectionSkipped)
619+ expected_failures = (
620+ BadResponseCode,
621+ ProberTimeout,
622+ ConnectionSkipped,
623+ InvalidHTTPSCertificate,
624+ InvalidHTTPSCertificateSkipped,
625+ )
626
627 def __init__(self, mirror, series, pocket, component, url, log_file):
628 self.mirror = mirror
629@@ -574,6 +718,8 @@ class MirrorCDImageProberCallbacks(LoggingMixin):
630 ProberTimeout,
631 RedirectToDifferentFile,
632 UnknownURLSchemeAfterRedirect,
633+ InvalidHTTPSCertificate,
634+ InvalidHTTPSCertificateSkipped,
635 )
636
637 def __init__(self, mirror, distroseries, flavour, log_file):
638diff --git a/lib/lp/registry/stories/webservice/xx-distribution-mirror.txt b/lib/lp/registry/stories/webservice/xx-distribution-mirror.txt
639index db17b40..3e49ebe 100644
640--- a/lib/lp/registry/stories/webservice/xx-distribution-mirror.txt
641+++ b/lib/lp/registry/stories/webservice/xx-distribution-mirror.txt
642@@ -23,6 +23,7 @@ mirrors:
643 enabled: True
644 ftp_base_url: None
645 http_base_url: u'http://archive.ubuntu.com/ubuntu/'
646+ https_base_url: None
647 name: u'canonical-archive'
648 official_candidate: True
649 owner_link: u'http://.../~mark'
650@@ -52,6 +53,7 @@ And CD image mirrors:
651 enabled: True
652 ftp_base_url: None
653 http_base_url: u'http://releases.ubuntu.com/'
654+ https_base_url: None
655 name: u'canonical-releases'
656 official_candidate: True
657 owner_link: u'http://.../~mark'
658@@ -145,6 +147,7 @@ Mirror listing admins may see all:
659 enabled: True
660 ftp_base_url: None
661 http_base_url: u'http://localhost:11375/archive-mirror/'
662+ https_base_url: None
663 name: u'archive-404-mirror'
664 official_candidate: True
665 owner_link: u'http://.../~name12'
666@@ -227,6 +230,7 @@ While others can be set with the appropriate authorization:
667 enabled: True
668 ftp_base_url: None
669 http_base_url: u'http://releases.ubuntu.com/'
670+ https_base_url: None
671 name: u'canonical-releases'
672 official_candidate: True
673 owner_link: u'http://.../~mark'
674diff --git a/lib/lp/registry/stories/webservice/xx-distribution.txt b/lib/lp/registry/stories/webservice/xx-distribution.txt
675index 1c0ed87..1486904 100644
676--- a/lib/lp/registry/stories/webservice/xx-distribution.txt
677+++ b/lib/lp/registry/stories/webservice/xx-distribution.txt
678@@ -159,6 +159,7 @@ packages matching (substring) the given text.
679 enabled: True
680 ftp_base_url: None
681 http_base_url: u'http://releases.ubuntu.com/'
682+ https_base_url: None
683 name: u'canonical-releases'
684 official_candidate: True
685 owner_link: u'http://.../~mark'
686diff --git a/lib/lp/registry/templates/distributionmirror-index.pt b/lib/lp/registry/templates/distributionmirror-index.pt
687index 78e934d..27f3139 100644
688--- a/lib/lp/registry/templates/distributionmirror-index.pt
689+++ b/lib/lp/registry/templates/distributionmirror-index.pt
690@@ -118,6 +118,10 @@
691 <h2>Mirror location information</h2>
692
693 <ul class="webref" id="mirror-urls">
694+ <li tal:condition="context/https_base_url" >
695+ <a tal:content="context/https_base_url"
696+ tal:attributes="href context/https_base_url">https://url/</a>
697+ </li>
698 <li tal:condition="context/http_base_url" >
699 <a tal:content="context/http_base_url"
700 tal:attributes="href context/http_base_url">http://url/</a>
701diff --git a/lib/lp/registry/templates/distributionmirror-macros.pt b/lib/lp/registry/templates/distributionmirror-macros.pt
702index 18f82af..489f770 100644
703--- a/lib/lp/registry/templates/distributionmirror-macros.pt
704+++ b/lib/lp/registry/templates/distributionmirror-macros.pt
705@@ -17,7 +17,7 @@
706 <tbody>
707 <tal:country_and_mirrors repeat="country_and_mirrors mirrors_by_country">
708 <tr class="head">
709- <th colspan="2"
710+ <th colspan="2"
711 tal:content="country_and_mirrors/country" />
712 <th tal:content="country_and_mirrors/throughput"/>
713 <th tal:condition="show_mirror_type">
714@@ -35,6 +35,8 @@
715 tal:content="mirror/title">Mirror Name</a>
716 </td>
717 <td>
718+ <a tal:condition="mirror/https_base_url"
719+ tal:attributes="href mirror/https_base_url">https</a>
720 <a tal:condition="mirror/http_base_url"
721 tal:attributes="href mirror/http_base_url">http</a>
722 <a tal:condition="mirror/ftp_base_url"
723diff --git a/lib/lp/registry/tests/distributionmirror_http_server.py b/lib/lp/registry/tests/distributionmirror_http_server.py
724index e3e0a39..d12c9b3 100644
725--- a/lib/lp/registry/tests/distributionmirror_http_server.py
726+++ b/lib/lp/registry/tests/distributionmirror_http_server.py
727@@ -1,6 +1,6 @@
728 #!/usr/bin/python
729 #
730-# Copyright 2009 Canonical Ltd. This software is licensed under the
731+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
732 # GNU Affero General Public License version 3 (see the file LICENSE).
733
734 from twisted.web.resource import Resource
735@@ -21,10 +21,10 @@ class DistributionMirrorTestHTTPServer(Resource):
736 :error: Respond with a '500 Internal Server Error' status.
737
738 :redirect-to-valid-mirror/*: Respond with a '302 Found' status,
739- redirecting to http://localhost:%(port)s/valid-mirror/*.
740+ redirecting to http(s)://localhost:%(port)s/valid-mirror/*.
741
742 :redirect-infinite-loop: Respond with a '302 Found' status, redirecting
743- to http://localhost:%(port)s/redirect-infinite-loop.
744+ to http(s)://localhost:%(port)s/redirect-infinite-loop.
745
746 :redirect-unknown-url-scheme: Respond with a '302 Found' status,
747 redirecting to ssh://localhost/redirect-unknown-url-scheme.
748@@ -32,11 +32,13 @@ class DistributionMirrorTestHTTPServer(Resource):
749 Any other path will cause the server to respond with a '404 Not Found'
750 status.
751 """
752+ protocol = "http"
753
754 def getChild(self, name, request):
755+ protocol = self.protocol
756 port = request.getHost().port
757 if name == 'valid-mirror':
758- leaf = DistributionMirrorTestHTTPServer()
759+ leaf = self.__class__()
760 leaf.isLeaf = True
761 return leaf
762 elif name == 'timeout':
763@@ -49,12 +51,14 @@ class DistributionMirrorTestHTTPServer(Resource):
764 'than one component.')
765 remaining_path = request.path.replace('/%s' % name, '')
766 leaf = RedirectingResource(
767- 'http://localhost:%s/valid-mirror%s' % (port, remaining_path))
768+ '%s://localhost:%s/valid-mirror%s' % (
769+ protocol, port, remaining_path))
770 leaf.isLeaf = True
771 return leaf
772 elif name == 'redirect-infinite-loop':
773 return RedirectingResource(
774- 'http://localhost:%s/redirect-infinite-loop' % port)
775+ '%s://localhost:%s/redirect-infinite-loop' %
776+ (protocol, port))
777 elif name == 'redirect-unknown-url-scheme':
778 return RedirectingResource(
779 'ssh://localhost/redirect-unknown-url-scheme')
780@@ -65,6 +69,11 @@ class DistributionMirrorTestHTTPServer(Resource):
781 return "Hi"
782
783
784+class DistributionMirrorTestSecureHTTPServer(DistributionMirrorTestHTTPServer):
785+ """HTTPS version of DistributionMirrorTestHTTPServer"""
786+ protocol = "https"
787+
788+
789 class RedirectingResource(Resource):
790
791 def __init__(self, redirection_url):
792@@ -85,4 +94,3 @@ class FiveHundredResource(Resource):
793 def render_GET(self, request):
794 request.setResponseCode(500)
795 request.write('ASPLODE!!!')
796- return NOT_DONE_YET
797diff --git a/lib/lp/registry/tests/test_distributionmirror_prober.py b/lib/lp/registry/tests/test_distributionmirror_prober.py
798index 39228d2..c3e363c 100644
799--- a/lib/lp/registry/tests/test_distributionmirror_prober.py
800+++ b/lib/lp/registry/tests/test_distributionmirror_prober.py
801@@ -1,4 +1,4 @@
802-# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
803+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
804 # GNU Affero General Public License version 3 (see the file LICENSE).
805
806 """distributionmirror-prober tests."""
807@@ -11,6 +11,7 @@ import logging
808 import os
809 from StringIO import StringIO
810
811+from fixtures import MockPatchObject
812 from lazr.uri import URI
813 import responses
814 from six.moves import http_client
815@@ -28,9 +29,14 @@ from testtools.twistedsupport import (
816 from twisted.internet import (
817 defer,
818 reactor,
819+ ssl,
820 )
821 from twisted.python.failure import Failure
822 from twisted.web import server
823+from twisted.web.client import (
824+ BrowserLikePolicyForHTTPS,
825+ ProxyAgent,
826+ )
827 from zope.component import getUtility
828 from zope.security.proxy import removeSecurityProxy
829
830@@ -44,6 +50,8 @@ from lp.registry.scripts.distributionmirror_prober import (
831 BadResponseCode,
832 ConnectionSkipped,
833 InfiniteLoopDetected,
834+ InvalidHTTPSCertificate,
835+ InvalidHTTPSCertificateSkipped,
836 LoggingMixin,
837 MAX_REDIRECTS,
838 MIN_REQUEST_TIMEOUT_RATIO,
839@@ -65,6 +73,7 @@ from lp.registry.scripts.distributionmirror_prober import (
840 )
841 from lp.registry.tests.distributionmirror_http_server import (
842 DistributionMirrorTestHTTPServer,
843+ DistributionMirrorTestSecureHTTPServer,
844 )
845 from lp.services.config import config
846 from lp.services.daemons.tachandler import TacTestSetup
847@@ -103,6 +112,186 @@ class HTTPServerTestSetup(TacTestSetup):
848 return os.path.join(self.root, 'distributionmirror_http_server.log')
849
850
851+
852+class LocalhostWhitelistedHTTPSPolicy(BrowserLikePolicyForHTTPS):
853+ """HTTPS policy that bypasses SSL certificate check when doing requests
854+ to localhost.
855+ """
856+
857+ def creatorForNetloc(self, hostname, port):
858+ # check if the hostname is in the the whitelist,
859+ # otherwise return the default policy
860+ if hostname == 'localhost':
861+ return ssl.CertificateOptions(verify=False)
862+ return super(LocalhostWhitelistedHTTPSPolicy, self).creatorForNetloc(
863+ hostname, port)
864+
865+
866+class TestProberHTTPSProtocolAndFactory(TestCase):
867+ layer = TwistedLayer
868+ run_tests_with = AsynchronousDeferredRunTestForBrokenTwisted.make_factory(
869+ timeout=30)
870+
871+ def setUp(self):
872+ super(TestProberHTTPSProtocolAndFactory, self).setUp()
873+ root = DistributionMirrorTestSecureHTTPServer()
874+ site = server.Site(root)
875+ site.displayTracebacks = False
876+ keys_path = os.path.join(config.root, "configs", "development")
877+ keys = ssl.DefaultOpenSSLContextFactory(
878+ os.path.join(keys_path, "launchpad.key"),
879+ os.path.join(keys_path, "launchpad.crt"),
880+ )
881+ self.listening_port = reactor.listenSSL(0, site, keys)
882+
883+ self.addCleanup(self.listening_port.stopListening)
884+
885+ # Change the default policy to accept localhost self-signed
886+ # certificates.
887+ original_probefactory_policy = ProberFactory.https_agent_policy
888+ original_redirect_policy = (
889+ RedirectAwareProberFactory.https_agent_policy)
890+ ProberFactory.https_agent_policy = LocalhostWhitelistedHTTPSPolicy
891+ RedirectAwareProberFactory.https_agent_policy = (
892+ LocalhostWhitelistedHTTPSPolicy)
893+
894+ for factory in (ProberFactory, RedirectAwareProberFactory):
895+ self.useFixture(MockPatchObject(
896+ factory, "https_agent_policy",
897+ LocalhostWhitelistedHTTPSPolicy))
898+
899+ self.port = self.listening_port.getHost().port
900+
901+ self.urls = {'timeout': u'https://localhost:%s/timeout' % self.port,
902+ '200': u'https://localhost:%s/valid-mirror' % self.port,
903+ '500': u'https://localhost:%s/error' % self.port,
904+ '404': u'https://localhost:%s/invalid-mirror' % self.port}
905+ self.pushConfig('launchpad', http_proxy=None)
906+
907+ self.useFixture(MockPatchObject(
908+ distributionmirror_prober, "host_requests", {}))
909+ self.useFixture(MockPatchObject(
910+ distributionmirror_prober, "host_timeouts", {}))
911+ self.useFixture(MockPatchObject(
912+ distributionmirror_prober, "invalid_certificate_hosts", set()))
913+
914+ def _createProberAndProbe(self, url):
915+ prober = ProberFactory(url)
916+ return prober.probe()
917+
918+ def test_timeout(self):
919+ prober = ProberFactory(self.urls['timeout'], timeout=0.5)
920+ d = prober.probe()
921+ return assert_fails_with(d, ProberTimeout)
922+
923+ def test_500(self):
924+ d = self._createProberAndProbe(self.urls['500'])
925+ return assert_fails_with(d, BadResponseCode)
926+
927+ def test_notfound(self):
928+ d = self._createProberAndProbe(self.urls['404'])
929+ return assert_fails_with(d, BadResponseCode)
930+
931+ def test_config_no_https_proxy(self):
932+ prober = ProberFactory(self.urls['200'])
933+ self.assertThat(prober, MatchesStructure.byEquality(
934+ request_scheme='https',
935+ request_host='localhost',
936+ request_port=self.port,
937+ request_path='/valid-mirror',
938+ connect_scheme='https',
939+ connect_host='localhost',
940+ connect_port=self.port,
941+ connect_path='/valid-mirror'))
942+
943+ def test_RedirectAwareProber_follows_https_redirect(self):
944+ url = 'https://localhost:%s/redirect-to-valid-mirror/file' % self.port
945+ prober = RedirectAwareProberFactory(url)
946+ self.assertEqual(prober.url, url)
947+ deferred = prober.probe()
948+
949+ def got_result(result):
950+ self.assertEqual(http_client.OK, result.code)
951+ self.assertEqual(
952+ 'https://localhost:%s/valid-mirror/file' % self.port,
953+ result.request.absoluteURI)
954+
955+ return deferred.addCallback(got_result)
956+
957+ def test_https_prober_uses_proxy(self):
958+ root = DistributionMirrorTestSecureHTTPServer()
959+ site = server.Site(root)
960+ proxy_listen_port = reactor.listenTCP(0, site)
961+ proxy_port = proxy_listen_port.getHost().port
962+ self.pushConfig(
963+ 'launchpad', http_proxy='http://localhost:%s/valid-mirror/file'
964+ % proxy_port)
965+
966+ url = 'https://localhost:%s/valid-mirror/file' % self.port
967+ prober = RedirectAwareProberFactory(url)
968+ self.assertEqual(prober.url, url)
969+ deferred = prober.probe()
970+
971+ def got_result(result):
972+ # We basically don't care about the result here. We just want to
973+ # check that it did the request to the correct URI,
974+ # and ProxyAgent was used pointing to the correct proxy.
975+ agent = prober.getHttpsClient()._agent
976+ self.assertIsInstance(agent, ProxyAgent)
977+ self.assertEqual('localhost', agent._proxyEndpoint._hostText)
978+ self.assertEqual(proxy_port, agent._proxyEndpoint._port)
979+
980+ self.assertEqual(
981+ 'https://localhost:%s/valid-mirror/file' % self.port,
982+ result.value.response.request.absoluteURI)
983+
984+ def cleanup(*args, **kwargs):
985+ proxy_listen_port.stopListening()
986+
987+ # Doing the proxy checks on the error callback because the
988+ # proxy is dummy and always returns 404.
989+ deferred.addErrback(got_result)
990+ deferred.addBoth(cleanup)
991+ return deferred
992+
993+ def test_https_fails_on_invalid_certificates(self):
994+ """Changes set back the default browser-like policy for HTTPS
995+ request and make sure the request is failing due to invalid
996+ (self-signed) certificate.
997+ """
998+ url = 'https://localhost:%s/valid-mirror/file' % self.port
999+ prober = RedirectAwareProberFactory(url)
1000+ prober.https_agent_policy = BrowserLikePolicyForHTTPS
1001+ self.assertEqual(prober.url, url)
1002+ deferred = prober.probe()
1003+
1004+ def on_failure(result):
1005+ self.assertIsInstance(result.value, InvalidHTTPSCertificate)
1006+ self.assertIn(
1007+ ("localhost", self.port),
1008+ distributionmirror_prober.invalid_certificate_hosts)
1009+
1010+ def on_success(result):
1011+ if result is not None:
1012+ self.fail(
1013+ "Should have raised SSL error. Got '%s' instead" % result)
1014+
1015+ deferred.addErrback(on_failure)
1016+ deferred.addCallback(on_success)
1017+ return deferred
1018+
1019+ def test_https_skips_invalid_certificates_hosts(self):
1020+ distributionmirror_prober.invalid_certificate_hosts.add(
1021+ ("localhost", self.port))
1022+ url = 'https://localhost:%s/valid-mirror/file' % self.port
1023+ prober = RedirectAwareProberFactory(url)
1024+ prober.https_agent_policy = BrowserLikePolicyForHTTPS
1025+ self.assertEqual(prober.url, url)
1026+ deferred = prober.probe()
1027+
1028+ return assert_fails_with(deferred, InvalidHTTPSCertificateSkipped)
1029+
1030+
1031 class TestProberProtocolAndFactory(TestCase):
1032
1033 layer = TwistedLayer
1034@@ -212,7 +401,7 @@ class TestProberProtocolAndFactory(TestCase):
1035 self.assertTrue(prober.url == new_url)
1036 self.assertTrue(result == str(http_client.OK))
1037
1038- return deferred.addCallback(got_result)
1039+ return deferred.addBoth(got_result)
1040
1041 def test_redirectawareprober_detects_infinite_loop(self):
1042 prober = RedirectAwareProberFactory(
1043@@ -737,12 +926,16 @@ class TestMirrorCDImageProberCallbacks(TestCaseWithFactory):
1044 ConnectionSkipped,
1045 RedirectToDifferentFile,
1046 UnknownURLSchemeAfterRedirect,
1047+ InvalidHTTPSCertificate,
1048+ InvalidHTTPSCertificateSkipped,
1049 ]))
1050 exceptions = [BadResponseCode(str(http_client.NOT_FOUND)),
1051 ProberTimeout('http://localhost/', 5),
1052 ConnectionSkipped(),
1053 RedirectToDifferentFile('/foo', '/bar'),
1054- UnknownURLSchemeAfterRedirect('https://localhost')]
1055+ UnknownURLSchemeAfterRedirect('https://localhost'),
1056+ InvalidHTTPSCertificate('localhost', 443),
1057+ InvalidHTTPSCertificateSkipped("https://localhost/xx")]
1058 for exception in exceptions:
1059 failure = callbacks.ensureOrDeleteMirrorCDImageSeries(
1060 [(defer.FAILURE, Failure(exception))])
1061diff --git a/lib/lp/scripts/utilities/importpedant.py b/lib/lp/scripts/utilities/importpedant.py
1062index ec41baf..c7859a4 100644
1063--- a/lib/lp/scripts/utilities/importpedant.py
1064+++ b/lib/lp/scripts/utilities/importpedant.py
1065@@ -1,4 +1,4 @@
1066-# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
1067+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
1068 # GNU Affero General Public License version 3 (see the file LICENSE).
1069
1070 from __future__ import absolute_import, print_function, unicode_literals
1071@@ -42,6 +42,8 @@ valid_imports_not_in_all = {
1072 'textwrap': set(['dedent']),
1073 'testtools.testresult.real': set(['_details_to_str']),
1074 'twisted.internet.threads': set(['deferToThreadPool']),
1075+ # Even docs tell us to use this class. See docs on WebClientContextFactory.
1076+ 'twisted.web.client': set(['BrowserLikePolicyForHTTPS']),
1077 'zope.component': set(
1078 ['adapter',
1079 'ComponentLookupError',
1080diff --git a/lib/lp/testing/factory.py b/lib/lp/testing/factory.py
1081index 85a05b6..6c0492f 100644
1082--- a/lib/lp/testing/factory.py
1083+++ b/lib/lp/testing/factory.py
1084@@ -2,7 +2,7 @@
1085 # NOTE: The first line above must stay first; do not move the copyright
1086 # notice to the top. See http://www.python.org/dev/peps/pep-0263/.
1087 #
1088-# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
1089+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
1090 # GNU Affero General Public License version 3 (see the file LICENSE).
1091
1092 """Testing infrastructure for the Launchpad application.
1093@@ -3527,13 +3527,13 @@ class BareLaunchpadObjectFactory(ObjectFactory):
1094 return proberecord
1095
1096 def makeMirror(self, distribution, displayname=None, country=None,
1097- http_url=None, ftp_url=None, rsync_url=None,
1098+ http_url=None, https_url=None, ftp_url=None, rsync_url=None,
1099 official_candidate=False):
1100 """Create a mirror for the distribution."""
1101 if displayname is None:
1102 displayname = self.getUniqueString("mirror")
1103 # If no URL is specified create an HTTP URL.
1104- if http_url is None and ftp_url is None and rsync_url is None:
1105+ if http_url is https_url is ftp_url is rsync_url is None:
1106 http_url = self.getUniqueURL()
1107 # If no country is given use Argentina.
1108 if country is None:
1109@@ -3547,6 +3547,7 @@ class BareLaunchpadObjectFactory(ObjectFactory):
1110 display_name=displayname,
1111 description=None,
1112 http_base_url=http_url,
1113+ https_base_url=https_url,
1114 ftp_base_url=ftp_url,
1115 rsync_base_url=rsync_url,
1116 official_candidate=official_candidate)