Merge ~hloeung/ubuntu-repository-cache:fixes into ubuntu-repository-cache:master

[Haw Loeung]

Allow overriding timeouts and drop it for single file sync to peers

[Paul Collins (pjdc)]

review: Approve (lgtm)

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Change successfully merged at revision 54d4058f2031d0b63ba4d97c96a3e483b04d47f8

diff --git a/lib/ubuntu_repository_cache/metadata_peers_sync.py b/lib/ubuntu_repository_cache/metadata_peers_sync.py
index 5be276f..7ab5bd1 100644
--- a/lib/ubuntu_repository_cache/metadata_peers_sync.py
+++ b/lib/ubuntu_repository_cache/metadata_peers_sync.py
@@ -53,14 +53,14 @@ def ubuntu_repository_cache_sync(environment, log):
 
     # Push ubuntu_next/ symlink to avoid metadata cleanup
     log('Pushing ubuntu_next symlink to peers')
-    mirror.rsync_to_peers(link_next, apache_data, peers=peers, log=log)
+    mirror.rsync_to_peers(link_next, apache_data, peers=peers, timeout=60, log=log)
 
     # Push the metadata to peers
     dest = '/'.join((apache_data, meta_ver))
     link_dir = '/'.join((apache_data, 'ubuntu_active'))
     log('Pushing updates to peers ({})'.format(meta_ver))
     source = meta_dir + '/'
-    successful_peers = mirror.rsync_to_peers(source, dest, link_dir=link_dir, peers=peers, log=log)
+    successful_peers = mirror.rsync_to_peers(source, dest, link_dir=link_dir, peers=peers, timeout=3600, log=log)
 
     if sorted(successful_peers) != sorted(peers):
         log('Failed to sync latest metadata to all peers, aborting')
@@ -69,12 +69,12 @@ def ubuntu_repository_cache_sync(environment, log):
     # Push metaversion_good canary to indicate a complete sync
     canary = '_'.join((meta_dir, 'good'))
     log('Pushing canary files to peers ({})'.format(canary))
-    mirror.rsync_to_peers(canary, apache_data, peers=successful_peers, log=log)
+    mirror.rsync_to_peers(canary, apache_data, peers=successful_peers, timeout=60, log=log)
 
     # Update ubuntu_active/ symlink
     log('Updating ubuntu_active symlink')
     util.update_active_symlink(os.path.basename(meta_dir), apache_root)
-    mirror.rsync_to_peers(link_dir, apache_data, peers=successful_peers, log=log)
+    mirror.rsync_to_peers(link_dir, apache_data, peers=successful_peers, timeout=60, log=log)
 
     duration = time.time() - start_time
     metric_name = 'ubuntu_repository_cache_metadata_sync_peers_duration'
diff --git a/lib/ubuntu_repository_cache/mirror.py b/lib/ubuntu_repository_cache/mirror.py
index cf07c4b..1cfa171 100644
--- a/lib/ubuntu_repository_cache/mirror.py
+++ b/lib/ubuntu_repository_cache/mirror.py
@@ -45,7 +45,7 @@ def rsync(from_path, to_path, flags='-r', options=None, timeout=None, log=None):
     return subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode('UTF-8').strip()
 
 
-def _rsync(source, dest, link_dest=None, optlist=None, user='www-sync', retries=1, log=None):
+def _rsync(source, dest, link_dest=None, optlist=None, user='www-sync', retries=1, timeout=4800, log=None):
     '''Perform an rsync as a specified user'''
 
     options = [
@@ -69,7 +69,7 @@ def _rsync(source, dest, link_dest=None, optlist=None, user='www-sync', retries=
     attempt = 0
     while True:
         try:
-            rsync(source, dest, options=options, timeout=4800, log=log)
+            rsync(source, dest, options=options, timeout=timeout, log=log)
         except CalledProcessError as excp:
             log('rsync failed({}): {}'.format(excp.returncode, excp.output))
             if not metadata_sync.rsync_should_giveup(excp):
@@ -88,7 +88,7 @@ def _rsync(source, dest, link_dest=None, optlist=None, user='www-sync', retries=
 
 
 @util.run_as_user('www-sync')
-def rsync_to_peers(source_dir, dest_dir, link_dir=None, user='www-sync', peers=None, log=None):
+def rsync_to_peers(source_dir, dest_dir, link_dir=None, user='www-sync', peers=None, timeout=4800, log=None):
     """
     Use rsync to push a directory to all configured peers.
 
@@ -117,7 +117,7 @@ def rsync_to_peers(source_dir, dest_dir, link_dir=None, user='www-sync', peers=N
         max_retries = 10
         for attempt in range(1, max_retries):
             try:
-                _rsync(source_dir, rsync_dest, link_dest=link_dir, user=user, retries=10, log=log)
+                _rsync(source_dir, rsync_dest, link_dest=link_dir, user=user, retries=10, timeout=timeout, log=log)
             except Exception as e:
                 util.send_to_influx(util.render_influx(metric_name, label, attempt))
                 failures = attempt
@@ -153,7 +153,7 @@ def rsync_to_peers(source_dir, dest_dir, link_dir=None, user='www-sync', peers=N
 
 
 @util.run_as_user('www-sync')
-def rsync_from_peer(source_dir, dest_dir, peer_ip, link_dir=None, user='www-sync', log=None):
+def rsync_from_peer(source_dir, dest_dir, peer_ip, link_dir=None, user='www-sync', timeout=4800, log=None):
     '''Use rsync to pull a directory from a configured peer.'''
 
     if not log:
@@ -161,7 +161,7 @@ def rsync_from_peer(source_dir, dest_dir, peer_ip, link_dir=None, user='www-sync
 
     rsync_source = '{}@{}:{}'.format(user, peer_ip, source_dir)
     log('Syncing {} to {}.'.format(rsync_source, dest_dir))
-    _rsync(rsync_source, dest_dir, link_dest=link_dir, user=user, retries=10, log=log)
+    _rsync(rsync_source, dest_dir, link_dest=link_dir, user=user, retries=10, timeout=timeout, log=log)
     log('Sync to {} complete.'.format(peer_ip))