Launchpad itself

Merge lp:~cjwatson/launchpad/mirror-prober-redirects into lp:launchpad

  1. mirror-prober-redirects
  2. Merge into devel
Proposed by Colin Watson
Status: Merged
Merged at revision: 19016
Proposed branch: lp:~cjwatson/launchpad/mirror-prober-redirects
Merge into: lp:launchpad
Diff against target: 146 lines (+22/-11)
5 files modified
database/sampledata/current-dev.sql (+1/-0)
database/sampledata/current.sql (+2/-1)
lib/lp/registry/doc/distribution-mirror.txt (+11/-6)
lib/lp/registry/scripts/distributionmirror_prober.py (+4/-2)
lib/lp/registry/stories/distributionmirror/xx-distribution-mirrors.txt (+4/-2)
To merge this branch: bzr merge lp:~cjwatson/launchpad/mirror-prober-redirects
Reviewer Review Type Date Requested Status
Tom Wardill (community) Approve
Review via email: mp+370493@code.launchpad.net

Commit message

Allow archive mirrors to return redirect responses.

Description of the change

APT used to not support these, but this was changed over ten years ago.

To post a comment you must log in.
Revision history for this message
Tom Wardill (twom) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'database/sampledata/current-dev.sql'
2--- database/sampledata/current-dev.sql 2019-05-22 14:57:45 +0000
3+++ database/sampledata/current-dev.sql 2019-07-23 14:58:40 +0000
4@@ -3443,6 +3443,7 @@
5 INSERT INTO distributionmirror (id, distribution, name, http_base_url, ftp_base_url, rsync_base_url, displayname, description, owner, speed, country, content, official_candidate, enabled, date_created, whiteboard, status, date_reviewed, reviewer, country_dns_mirror) VALUES (8, 1, 'canonical-archive', 'http://archive.ubuntu.com/ubuntu/', NULL, NULL, NULL, NULL, 1, 70, 225, 1, true, true, '2006-10-16 18:31:43.434567', NULL, 30, NULL, NULL, false);
6 INSERT INTO distributionmirror (id, distribution, name, http_base_url, ftp_base_url, rsync_base_url, displayname, description, owner, speed, country, content, official_candidate, enabled, date_created, whiteboard, status, date_reviewed, reviewer, country_dns_mirror) VALUES (9, 1, 'canonical-releases', 'http://releases.ubuntu.com/', NULL, NULL, NULL, NULL, 1, 70, 225, 2, true, true, '2006-10-16 18:31:43.434567', NULL, 30, NULL, NULL, false);
7 INSERT INTO distributionmirror (id, distribution, name, http_base_url, ftp_base_url, rsync_base_url, displayname, description, owner, speed, country, content, official_candidate, enabled, date_created, whiteboard, status, date_reviewed, reviewer, country_dns_mirror) VALUES (10, 1, 'random-releases-mirror', 'http://releases.random.com/', NULL, NULL, NULL, NULL, 1, 70, 225, 2, true, true, '2006-10-16 18:31:43.434567', NULL, 10, NULL, NULL, false);
8+INSERT INTO distributionmirror (id, distribution, name, http_base_url, ftp_base_url, rsync_base_url, displayname, description, owner, speed, country, content, official_candidate, enabled, date_created, whiteboard, status, date_reviewed, reviewer, country_dns_mirror) VALUES (11, 1, 'archive-redirect-mirror', 'http://localhost:11375/redirect-to-valid-mirror/', NULL, NULL, NULL, NULL, 1, 10, 9, 1, true, true, '2019-07-23 13:16:51.953354', NULL, 30, NULL, NULL, false);
9
10
11 ALTER TABLE distributionmirror ENABLE TRIGGER ALL;
12
13=== modified file 'database/sampledata/current.sql'
14--- database/sampledata/current.sql 2019-05-22 14:57:45 +0000
15+++ database/sampledata/current.sql 2019-07-23 14:58:40 +0000
16@@ -1,4 +1,4 @@
17--- Copyright 2010-2017 Canonical Ltd. This software is licensed under the
18+-- Copyright 2010-2019 Canonical Ltd. This software is licensed under the
19 -- GNU Affero General Public License version 3 (see the file LICENSE).
20 -- Created using pg_dump (PostgreSQL) 9.3.5
21
22@@ -3380,6 +3380,7 @@
23 INSERT INTO distributionmirror (id, distribution, name, http_base_url, ftp_base_url, rsync_base_url, displayname, description, owner, speed, country, content, official_candidate, enabled, date_created, whiteboard, status, date_reviewed, reviewer, country_dns_mirror) VALUES (8, 1, 'canonical-archive', 'http://archive.ubuntu.com/ubuntu/', NULL, NULL, NULL, NULL, 1, 70, 225, 1, true, true, '2006-10-16 18:31:43.434567', NULL, 30, NULL, NULL, false);
24 INSERT INTO distributionmirror (id, distribution, name, http_base_url, ftp_base_url, rsync_base_url, displayname, description, owner, speed, country, content, official_candidate, enabled, date_created, whiteboard, status, date_reviewed, reviewer, country_dns_mirror) VALUES (9, 1, 'canonical-releases', 'http://releases.ubuntu.com/', NULL, NULL, NULL, NULL, 1, 70, 225, 2, true, true, '2006-10-16 18:31:43.434567', NULL, 30, NULL, NULL, false);
25 INSERT INTO distributionmirror (id, distribution, name, http_base_url, ftp_base_url, rsync_base_url, displayname, description, owner, speed, country, content, official_candidate, enabled, date_created, whiteboard, status, date_reviewed, reviewer, country_dns_mirror) VALUES (10, 1, 'random-releases-mirror', 'http://releases.random.com/', NULL, NULL, NULL, NULL, 1, 70, 225, 2, true, true, '2006-10-16 18:31:43.434567', NULL, 10, NULL, NULL, false);
26+INSERT INTO distributionmirror (id, distribution, name, http_base_url, ftp_base_url, rsync_base_url, displayname, description, owner, speed, country, content, official_candidate, enabled, date_created, whiteboard, status, date_reviewed, reviewer, country_dns_mirror) VALUES (11, 1, 'archive-redirect-mirror', 'http://localhost:11375/redirect-to-valid-mirror/', NULL, NULL, NULL, NULL, 1, 10, 9, 1, true, true, '2019-07-23 13:16:51.953354', NULL, 30, NULL, NULL, false);
27
28
29 ALTER TABLE distributionmirror ENABLE TRIGGER ALL;
30
31=== modified file 'lib/lp/registry/doc/distribution-mirror.txt'
32--- lib/lp/registry/doc/distribution-mirror.txt 2019-05-22 14:57:45 +0000
33+++ lib/lp/registry/doc/distribution-mirror.txt 2019-07-23 14:58:40 +0000
34@@ -122,7 +122,8 @@
35 [(u'canonical-archive', '100 Mbps', 'OFFICIAL'),
36 (u'archive-404-mirror', '512 Kbps', 'OFFICIAL'),
37 (u'archive-mirror', '128 Kbps', 'OFFICIAL'),
38- (u'archive-mirror2', '128 Kbps', 'OFFICIAL')]
39+ (u'archive-mirror2', '128 Kbps', 'OFFICIAL'),
40+ (u'archive-redirect-mirror', '128 Kbps', 'OFFICIAL')]
41
42 >>> [(mirror.name, mirror.speed.title, mirror.status.name)
43 ... for mirror in ubuntu.cdimage_mirrors]
44@@ -242,7 +243,7 @@
45 >>> [mirror.name
46 ... for mirror in mirrorset.getMirrorsToProbe(MirrorContent.ARCHIVE)]
47 [u'canonical-archive', u'archive-404-mirror', u'archive-mirror',
48- u'archive-mirror2']
49+ u'archive-mirror2', u'archive-redirect-mirror']
50
51 >>> [mirror.name
52 ... for mirror in mirrorset.getMirrorsToProbe(MirrorContent.RELEASE)]
53@@ -263,7 +264,8 @@
54
55 >>> [mirror.name
56 ... for mirror in mirrorset.getMirrorsToProbe(MirrorContent.ARCHIVE)]
57- [u'canonical-archive', u'archive-404-mirror', u'archive-mirror2']
58+ [u'canonical-archive', u'archive-404-mirror', u'archive-mirror2',
59+ u'archive-redirect-mirror']
60
61 >>> [mirror.name
62 ... for mirror in mirrorset.getMirrorsToProbe(MirrorContent.RELEASE)]
63@@ -278,7 +280,7 @@
64 ... MirrorContent.ARCHIVE, ignore_last_probe=True)
65 >>> [mirror.name for mirror in mirrors]
66 [u'canonical-archive', u'archive-404-mirror', u'archive-mirror',
67- u'archive-mirror2']
68+ u'archive-mirror2', u'archive-redirect-mirror']
69
70 It's also possible to limit the number of mirrors we want to probe by
71 passing an extra limit= argument to mirrorset.getMirrorsToProbe(). If
72@@ -306,9 +308,10 @@
73 archive-404-mirror NEVER
74 archive-mirror TODAY
75 archive-mirror2 2006-05-24...
76+ archive-redirect-mirror NEVER
77
78 >>> mirrors = mirrorset.getMirrorsToProbe(
79- ... MirrorContent.ARCHIVE, ignore_last_probe=True, limit=3)
80+ ... MirrorContent.ARCHIVE, ignore_last_probe=True, limit=4)
81 >>> for mirror in mirrors:
82 ... last_probe = mirror.last_probe_record
83 ... last_probe_date = 'NEVER'
84@@ -318,6 +321,7 @@
85 canonical-archive NEVER
86 archive-404-mirror NEVER
87 archive-mirror2 2006-05-24...
88+ archive-redirect-mirror NEVER
89
90
91 If when we finish probing a mirror, that mirror doesn't have any
92@@ -400,6 +404,7 @@
93 getExpectedPackagesPaths() and getExpectedSourcesPaths() methods of
94 IDistributionMirror.
95
96+ >>> mirror = mirrorset.getByName('archive-mirror2')
97 >>> paths = mirror.getExpectedPackagesPaths()
98 >>> [path for (series, pocket, component, path) in paths]
99 [u'dists/breezy-autotest/main/binary-i386/Packages.gz',
100@@ -646,7 +651,7 @@
101 >>> print stderr
102 INFO Creating lockfile: /var/lock/launchpad-distributionmirror-prober.lock
103 INFO Probing Archive Mirrors
104- INFO Probed 4 mirrors.
105+ INFO Probed 5 mirrors.
106 INFO Disabling 1 mirror(s):
107 http://launchpad.test/ubuntu/+mirror/archive-404-mirror
108 INFO Re-enabling 1 mirror(s):
109
110=== modified file 'lib/lp/registry/scripts/distributionmirror_prober.py'
111--- lib/lp/registry/scripts/distributionmirror_prober.py 2018-07-13 12:48:19 +0000
112+++ lib/lp/registry/scripts/distributionmirror_prober.py 2019-07-23 14:58:40 +0000
113@@ -1,4 +1,4 @@
114-# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
115+# Copyright 2009-2019 Canonical Ltd. This software is licensed under the
116 # GNU Affero General Public License version 3 (see the file LICENSE).
117
118 __metaclass__ = type
119@@ -688,7 +688,9 @@
120 callbacks = ArchiveMirrorProberCallbacks(
121 mirror, series, pocket, component, url, logfile)
122 unchecked_keys.append(url)
123- prober = ProberFactory(url)
124+ # APT has supported redirects since 0.7.21 (2009-04-14), so allow
125+ # them here too.
126+ prober = RedirectAwareProberFactory(url)
127
128 deferred = request_manager.run(prober.request_host, prober.probe)
129 deferred.addCallbacks(
130
131=== modified file 'lib/lp/registry/stories/distributionmirror/xx-distribution-mirrors.txt'
132--- lib/lp/registry/stories/distributionmirror/xx-distribution-mirrors.txt 2019-05-22 14:57:45 +0000
133+++ lib/lp/registry/stories/distributionmirror/xx-distribution-mirrors.txt 2019-07-23 14:58:40 +0000
134@@ -42,8 +42,10 @@
135 >>> print browser.title
136 Mirrors :...
137 >>> print_mirrors_by_countries(browser.contents)
138- Antarctica: [(u'Archive-mirror2', u'http', u'128 Kbps',
139- u'Six hours behind')]
140+ Antarctica:
141+ [(u'Archive-mirror2', u'http', u'128 Kbps', u'Six hours behind'),
142+ (u'Archive-redirect-mirror', u'http', u'128 Kbps',
143+ u'Last update unknown')]
144 France:
145 [(u'Archive-404-mirror', u'http', u'512 Kbps', u'Last update unknown'),
146 (u'Archive-mirror', u'http', u'128 Kbps', u'Last update unknown')]