Merge lp:~daniel-thewatkins/charms/trusty/ubuntu-repository-cache/add-rsync-timeout into lp:charms/trusty/ubuntu-repository-cache

[Dan Watkins]

This does a couple of rsync-related things: it adds a 120 second timeout (which should deal with the hangs seen in the attached) bug, and also firms up the logic for syncing to peers to avoid telling a failed peer it was successful.

Specifically, we have three phases to the peer update:
 (1) transfer the mirror payload to all peers,
 (2) transfer a file to each peer that tells them that they have a good version of the files, and then
 (3) change a relationship variable, to trigger the actual switch.

This ensures that if a transfer in (1) fails, then that peer won't be included in the transfer for (2).

[Adam Israel (aisrael)]

Hi Dan,

I had a chance to review this MP. The changes here look good, and you have a +1 from me on that, with the caveat that it's dependent on lp:262072 and lp:261356 being merged.

review: Approve

[Robert C Jennings (rcj)]

Reviewed and the changes look clean. I have not had a chance to test independently.

review: Approve

=== modified file 'hooks/hooks.py'
--- hooks/hooks.py	2015-02-24 15:41:56 +0000
+++ hooks/hooks.py	2015-06-08 16:18:05 +0000
@@ -84,12 +84,13 @@
         link_dir = '/'.join((apache_data, 'ubuntu_active'))
         LOG('Pushing updates to peers', hookenv.DEBUG)
         source = meta_dir + '/'
-        mirror.rsync_to_peers(source, dest, link_dir=link_dir)
+        successful_peers = mirror.rsync_to_peers(
+            source, dest, link_dir=link_dir)
 
         # Push metaversion_good canary to indicate a complete sync
         canary = '_'.join((meta_dir, 'good'))
         LOG('Pushing canary files to peers', hookenv.DEBUG)
-        mirror.rsync_to_peers(canary, apache_data)
+        mirror.rsync_to_peers(canary, apache_data, peers=successful_peers)
 
         # Update 'meta_version' relation setting
         LOG('', hookenv.DEBUG)

=== modified file 'lib/ubuntu_repository_cache/mirror.py'
--- lib/ubuntu_repository_cache/mirror.py	2015-03-13 15:29:57 +0000
+++ lib/ubuntu_repository_cache/mirror.py	2015-06-08 16:18:05 +0000
@@ -5,7 +5,9 @@
 import glob
 import os
 import socket
+import sys
 import time
+import traceback
 
 from collections import OrderedDict
 from datetime import datetime
@@ -330,6 +332,7 @@
                '--compress',
                '--delete',
                '--delete-during',
+               '--timeout=120',
                '-e', 'sudo -u {} ssh -l {}'.format(user, user),
                ]
     if optlist:
@@ -364,21 +367,34 @@
 @util.run_as_user('www-sync')
 def rsync_to_peers(source_dir, dest_dir, link_dir=None, user='www-sync',
                    peers=None):
-    '''Use rsync to push a directory to all configured peer.
+    """
+    Use rsync to push a directory to all configured peers.
 
     Peers are configured in the 'cluster' relationship using
-    charmhelpers.contrib.unison.ssh_authorized_peers().
-    '''
-
-    if not peers:
+    charmhelpers.contrib.unison.ssh_authorized_peers(); this list can be
+    overridden with the peers parameter.
+
+    A list of the peers whose transfers were successful will be
+    returned.
+    """
+
+    if peers is None:
         peers = unison.collect_authed_hosts(peer_interface='cluster')
 
+    successful_peers = []
     for peer in peers:
         rsync_dest = '{}@{}:{}'.format(user, peer, dest_dir)
         LOG('Syncing {} to {}.'.format(source_dir, rsync_dest))
-        _rsync(source_dir, rsync_dest, link_dest=link_dir,
-               user=user, retries=10)
-        LOG('Sync to {} complete.'.format(peer))
+        try:
+            _rsync(source_dir, rsync_dest, link_dest=link_dir,
+                   user=user, retries=10)
+        except Exception:
+            LOG('Sync to {} failed'.format(peer))
+            traceback.print_exc(file=sys.stdout)
+        else:
+            successful_peers.append(peer)
+            LOG('Sync to {} complete.'.format(peer))
+    return successful_peers
 
 
 @util.run_as_user('www-sync')
@@ -505,11 +521,17 @@
                 active_dir = '/'.join((apache_data, 'ubuntu_active'))
                 source = meta_dir + '/'
                 dest = meta_dir
-                rsync_from_peer(source, dest,
-                                leader_rel.get('private-address'),
-                                link_dir=active_dir)
-                util.update_active_symlink(meta_ver)
-                service.unpause()
+                try:
+                    rsync_from_peer(source, dest,
+                                    leader_rel.get('private-address'),
+                                    link_dir=active_dir)
+                except CalledProcessError:
+                    traceback.print_exc()
+                    LOG('Failed to pull metadata from leader; service stopped'
+                        ' until next update from leader.')
+                else:
+                    util.update_active_symlink(meta_ver)
+                    service.unpause()
 
             # Do nothing. Service is stopped until next update from leader
             else: