Merge lp:~clint-fewbar/txzookeeper/backoff-retry-on-fail into lp:txzookeeper

[Clint Byrum]

This morning the Zookeeper server serving about 45 boxes for juju agents crashed and started recovering. They spewed ConnectionLost errors at an incredibly high rate, pounding on the zookeeper and making it worse. This code backs off the retries so the server can have some breathing room to recover.

Unmerged revisions

52. By Clint Byrum on 2012-10-30

back off retry rate on connection problems to prevent dog-piling on a dead server

51. By Clint Byrum on 2012-10-30

Handle ConnectionLossException more gracefully

=== modified file 'txzookeeper/managed.py'
--- txzookeeper/managed.py	2012-09-19 12:54:25 +0000
+++ txzookeeper/managed.py	2012-10-30 14:27:21 +0000
@@ -154,6 +154,8 @@
         self._session_notifications = []
         self._reconnect_lock = DeferredLock()
         self.set_connection_error_callback(self._cb_connection_error)
+        self._backoff_seconds = 0
+        self._max_backoff_seconds = 60
 
     def subscribe_new_session(self):
         d = Deferred()
@@ -204,6 +206,10 @@
     def _cb_restablish_session(self):
         """Re-establish a new session, and recreate ephemerals and watches.
         """
+        # If we have some failures, back off
+        if self._backoff_seconds:
+            yield deferLater(reactor, self._backoff_seconds, lambda: none)
+
         # Reconnect
         while 1:
             if self.handle is None:
@@ -234,6 +240,9 @@
         notifications = self._session_notifications
         self._session_notifications = []
 
+        # all good, reset backoff
+        self._backoff_seconds = 0
+
         for n in notifications:
             n.callback(True)
 
@@ -252,9 +261,12 @@
         """
         if not isinstance(error, (
             zookeeper.SessionExpiredException,
+            zookeeper.ConnectionLossException,
             NotConnectedException,
             zookeeper.ClosingException)):
             raise error
+        if self._backoff_seconds < self._max_backoff:
+            self._backoff_seconds += 10
         yield self._cb_restablish_session()
         raise zookeeper.ConnectionLossException