Merge lp:~cjwatson/launchpad/mirror-prober-extra-slashes into lp:launchpad

[Colin Watson]

Remove extra internal slashes from URLs requested by the mirror prober.

[William Grant (wgrant)]

review: Approve (code)

=== modified file 'lib/lp/registry/scripts/distributionmirror_prober.py'
--- lib/lp/registry/scripts/distributionmirror_prober.py	2016-11-11 13:00:06 +0000
+++ lib/lp/registry/scripts/distributionmirror_prober.py	2017-05-22 09:34:40 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2010 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2017 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 __metaclass__ = type
@@ -651,7 +651,7 @@
     for line in _get_cdimage_file_list().readlines():
         flavour, seriesname, path, size = line.split('\t')
         paths = d.setdefault((flavour, seriesname), [])
-        paths.append(path)
+        paths.append(path.lstrip('/'))
 
     ubuntu = getUtility(ILaunchpadCelebrities).ubuntu
     paths = []
@@ -681,12 +681,15 @@
     publishing time are available on that mirror, giving us an idea of when it
     was last synced to the main archive.
     """
+    base_url = mirror.base_url
+    if not base_url.endswith('/'):
+        base_url += '/'
     packages_paths = mirror.getExpectedPackagesPaths()
     sources_paths = mirror.getExpectedSourcesPaths()
     all_paths = itertools.chain(packages_paths, sources_paths)
     request_manager = RequestManager()
     for series, pocket, component, path in all_paths:
-        url = "%s/%s" % (mirror.base_url, path)
+        url = urlparse.urljoin(base_url, path)
         callbacks = ArchiveMirrorProberCallbacks(
             mirror, series, pocket, component, url, logfile)
         unchecked_keys.append(url)
@@ -710,6 +713,10 @@
     files for a given series and flavour, then we consider that mirror is
     actually mirroring that series and flavour.
     """
+    base_url = mirror.base_url
+    if not base_url.endswith('/'):
+        base_url += '/'
+
     # The list of files a mirror should contain will change over time and we
     # don't want to keep records for files a mirror doesn't need to have
     # anymore, so we delete all records before start probing. This also fixes
@@ -730,7 +737,7 @@
         deferredList = []
         request_manager = RequestManager()
         for path in paths:
-            url = '%s/%s' % (mirror.base_url, path)
+            url = urlparse.urljoin(base_url, path)
             # Use a RedirectAwareProberFactory because CD mirrors are allowed
             # to redirect, and we need to cope with that.
             prober = RedirectAwareProberFactory(url)