Merge ~hloeung/ubuntu-repository-cache:master into ubuntu-repository-cache:master

[Haw Loeung]

Make it more resilient around DNS failures on upstream metadata sync

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[Thomas Cuthbert (tcuthbert)]

LGTM +1 with some comments inline that are nonblocking.

review: Approve

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Change successfully merged at revision 2d0d0fb344d4047121007fccd8debee545299731

diff --git a/lib/ubuntu_repository_cache/metadata_sync.py b/lib/ubuntu_repository_cache/metadata_sync.py
index 8cdb85f..b2da773 100644
--- a/lib/ubuntu_repository_cache/metadata_sync.py
+++ b/lib/ubuntu_repository_cache/metadata_sync.py
@@ -152,9 +152,23 @@ def mirror_archive(hostname, apache_root, local_unit, log, mirror_series, rsync_
     dest = '/'.join((apache_data, 'ubuntu_{}_u{}'.format(now.isoformat(sep='_'), local_unit.split('/')[-1])))
     log('Performing sync to %s' % dest)
 
-    # The archive may have multiple IP addresses, try each several times.
-    archive_ips = [i for i in socket.gethostbyname_ex(hostname)[-1]]
-    archive_ip_iterator = cycle(archive_ips)
+    max_retries = 5
+    for attempt in range(1, max_retries):
+        try:
+             # The archive may have multiple IP addresses, try each several times.
+             archive_ips = [i for i in socket.gethostbyname_ex(hostname)[-1]]
+             archive_ip_iterator = cycle(archive_ips)
+        except socket.gaierror as e:
+            # Save the exact exception and reuse when we need to
+            # reraise it after exhausting all available attempts.
+            exp = e
+            # Exponential backoff.
+            sleep = random.randint(attempt, 10 * attempt)
+            time.sleep(sleep)
+        else:
+            break
+    else:
+        raise exp
 
     content_mirrored = False
 
@@ -400,13 +414,13 @@ def clean_metadata_dir(apache_root, log, keep=None):
             keep.add(entry)
             count += 1
 
+    # We want to make sure it isn't the directory
+    # ubuntu_active points to (LP#1799595).
+    active = os.path.realpath('/'.join((apache_data, 'ubuntu_active')))
+    # We also want to make sure it isn't ubuntu_next, in-progress sync.
+    ubuntu_next = os.path.realpath('/'.join((apache_data, 'ubuntu_next')))
     for entry in [os.path.realpath(p) for p in glob.glob(glob_path)]:
         if entry not in keep:
-            # We want to make sure it isn't the directory
-            # ubuntu_active points to (LP#1799595).
-            active = os.path.realpath('/'.join((apache_data, 'ubuntu_active')))
-            # We also want to make sure it isn't ubuntu_next, in-progress sync.
-            ubuntu_next = os.path.realpath('/'.join((apache_data, 'ubuntu_next')))
             if entry == active:
                 log('ubuntu_active points to {}, skipping'.format(entry))
             elif entry == ubuntu_next: