Merge lp:~cjwatson/launchpad/reduce-mp-timeouts into lp:launchpad

[Colin Watson]

Reduce timeout for Git hosting calls when rendering merge proposals.

Reduce timeout for Git hosting calls when rendering merge proposals. This means that we can cope with the backend taking longer than our timeout in some edge cases, falling back to leaving those parts of the rendered page empty.

This is based on an approach already used for fetching diffs from the librarian, but I took the opportunity to turn that into a generic reduced_timeout context manager since it seems likely that we'll find uses for it elsewhere.

[Maximiliano Bertacchini (maxiberta)]

Looks good to me! Thanks

review: Approve

=== modified file 'lib/lp/code/browser/branchmergeproposal.py'
--- lib/lp/code/browser/branchmergeproposal.py	2016-05-14 09:54:32 +0000
+++ lib/lp/code/browser/branchmergeproposal.py	2016-07-01 22:03:52 +0000
@@ -107,6 +107,11 @@
     cachedproperty,
     get_property_cache,
     )
+from lp.services.scripts import log
+from lp.services.timeout import (
+    reduced_timeout,
+    TimeoutError,
+    )
 from lp.services.webapp import (
     canonical_url,
     ContextMenu,
@@ -343,7 +348,17 @@
     @cachedproperty
     def unlanded_revisions(self):
         """Return the unlanded revisions from the source branch."""
-        return self.context.getUnlandedSourceBranchRevisions()
+        with reduced_timeout(1.0, webapp_max=5.0):
+            try:
+                return self.context.getUnlandedSourceBranchRevisions()
+            except TimeoutError:
+                log.exception(
+                    "Timeout fetching unlanded source revisions for merge "
+                    "proposal %s (%s => %s)" % (
+                        self.context.id,
+                        self.context.merge_source.identity,
+                        self.context.merge_target.identity))
+                return []
 
     @property
     def pending_writes(self):
@@ -615,7 +630,17 @@
         """Return a conversation that is to be rendered."""
         # Sort the comments by date order.
         merge_proposal = self.context
-        groups = list(merge_proposal.getRevisionsSinceReviewStart())
+        with reduced_timeout(1.0, webapp_max=5.0):
+            try:
+                groups = list(merge_proposal.getRevisionsSinceReviewStart())
+            except TimeoutError:
+                log.exception(
+                    "Timeout fetching revisions since review start for "
+                    "merge proposal %s (%s => %s)" % (
+                        merge_proposal.id,
+                        merge_proposal.merge_source.identity,
+                        merge_proposal.merge_target.identity))
+                groups = []
         source = merge_proposal.merge_source
         if IBranch.providedBy(source):
             source = DecoratedBranch(source)

=== modified file 'lib/lp/code/browser/tests/test_branchmergeproposal.py'
--- lib/lp/code/browser/tests/test_branchmergeproposal.py	2016-06-03 14:08:52 +0000
+++ lib/lp/code/browser/tests/test_branchmergeproposal.py	2016-07-01 22:03:52 +0000
@@ -14,6 +14,7 @@
 import hashlib
 import re
 
+from fixtures import FakeLogger
 from lazr.lifecycle.event import ObjectModifiedEvent
 from lazr.restful.interfaces import IJSONRequestCache
 import pytz
@@ -77,6 +78,7 @@
     )
 from lp.services.librarian.interfaces.client import LibrarianServerError
 from lp.services.messages.model.message import MessageSet
+from lp.services.timeout import TimeoutError
 from lp.services.webapp import canonical_url
 from lp.services.webapp.interfaces import BrowserNotificationLevel
 from lp.services.webapp.servers import LaunchpadTestRequest
@@ -1914,6 +1916,16 @@
         self.assertThat(browser.contents, Not(has_read_more))
         self.assertEqual(mp_url, browser.url)
 
+    def test_timeout(self):
+        """The page renders even if fetching revisions times out."""
+        self.useFixture(FakeLogger())
+        bmp = self.factory.makeBranchMergeProposalForGit()
+        comment = self.factory.makeCodeReviewComment(
+            body='x y' * 100, merge_proposal=bmp)
+        self.hosting_client.getLog.failure = TimeoutError
+        browser = self.getViewBrowser(comment.branch_merge_proposal)
+        self.assertIn('x y' * 100, browser.contents)
+
 
 class TestLatestProposalsForEachBranchMixin:
     """Confirm that the latest branch is returned."""

=== modified file 'lib/lp/code/model/diff.py'
--- lib/lp/code/model/diff.py	2015-08-28 15:04:26 +0000
+++ lib/lp/code/model/diff.py	2016-07-01 22:03:52 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2015 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2016 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Implementation classes for IDiff, etc."""
@@ -58,7 +58,10 @@
     LIBRARIAN_SERVER_DEFAULT_TIMEOUT,
     )
 from lp.services.propertycache import get_property_cache
-from lp.services.webapp.adapter import get_request_remaining_seconds
+from lp.services.timeout import (
+    get_default_timeout_function,
+    reduced_timeout,
+    )
 
 
 @implementer(IDiff)
@@ -97,32 +100,16 @@
         if self.diff_text is None:
             return ''
         else:
-            self.diff_text.open(self._getDiffTimeout())
+            with reduced_timeout(
+                    0.01, webapp_max=2.0,
+                    default=LIBRARIAN_SERVER_DEFAULT_TIMEOUT):
+                timeout = get_default_timeout_function()()
+            self.diff_text.open(timeout)
             try:
                 return self.diff_text.read(config.diff.max_read_size)
             finally:
                 self.diff_text.close()
 
-    def _getDiffTimeout(self):
-        """Return the seconds allocated to get the diff from the librarian.
-
-         the value will be Non for scripts, 2 for the webapp, or if thre is
-         little request time left, the number will be smaller or equal to
-         the remaining request time.
-        """
-        remaining = get_request_remaining_seconds()
-        if remaining is None:
-            return LIBRARIAN_SERVER_DEFAULT_TIMEOUT
-        elif remaining > 2.0:
-            # The maximum permitted time for webapp requests.
-            return 2.0
-        elif remaining > 0.01:
-            # Shave off 1 hundreth of a second off so that the call site
-            # has a chance to recover.
-            return remaining - 0.01
-        else:
-            return remaining
-
     @property
     def oversized(self):
         # If the size of the content of the librarian file is over the

=== modified file 'lib/lp/code/model/githosting.py'
--- lib/lp/code/model/githosting.py	2016-05-19 14:54:33 +0000
+++ lib/lp/code/model/githosting.py	2016-07-01 22:03:52 +0000
@@ -9,11 +9,13 @@
     ]
 
 import json
+import sys
 from urllib import quote
 from urlparse import urljoin
 
 from lazr.restful.utils import get_current_browser_request
 import requests
+from six import reraise
 from zope.interface import implementer
 
 from lp.code.errors import (
@@ -24,11 +26,14 @@
 from lp.code.interfaces.githosting import IGitHostingClient
 from lp.services.config import config
 from lp.services.timeline.requesttimeline import get_request_timeline
-from lp.services.timeout import urlfetch
-
-
-class HTTPResponseNotOK(Exception):
-    pass
+from lp.services.timeout import (
+    TimeoutError,
+    urlfetch,
+    )
+
+
+class RequestExceptionWrapper(requests.RequestException):
+    """A non-requests exception that occurred during a request."""
 
 
 @implementer(IGitHostingClient)
@@ -46,10 +51,18 @@
             response = urlfetch(
                 urljoin(self.endpoint, path), trust_env=False, method=method,
                 **kwargs)
-        except requests.HTTPError as e:
-            raise HTTPResponseNotOK(e.response.content)
+        except TimeoutError:
+            # Re-raise this directly so that it can be handled specially by
+            # callers.
+            raise
+        except Exception:
+            _, val, tb = sys.exc_info()
+            reraise(
+                RequestExceptionWrapper, RequestExceptionWrapper(*val.args),
+                tb)
         finally:
             action.finish()
+        response.raise_for_status()
         if response.content:
             return response.json()
         else:
@@ -75,7 +88,7 @@
             else:
                 request = {"repo_path": path}
             self._post("/repo", json=request)
-        except Exception as e:
+        except requests.RequestException as e:
             raise GitRepositoryCreationFault(
                 "Failed to create Git repository: %s" % unicode(e))
 
@@ -83,7 +96,7 @@
         """See `IGitHostingClient`."""
         try:
             return self._get("/repo/%s" % path)
-        except Exception as e:
+        except requests.RequestException as e:
             raise GitRepositoryScanFault(
                 "Failed to get properties of Git repository: %s" % unicode(e))
 
@@ -91,7 +104,7 @@
         """See `IGitHostingClient`."""
         try:
             self._patch("/repo/%s" % path, json=props)
-        except Exception as e:
+        except requests.RequestException as e:
             raise GitRepositoryScanFault(
                 "Failed to set properties of Git repository: %s" % unicode(e))
 
@@ -99,7 +112,7 @@
         """See `IGitHostingClient`."""
         try:
             return self._get("/repo/%s/refs" % path)
-        except Exception as e:
+        except requests.RequestException as e:
             raise GitRepositoryScanFault(
                 "Failed to get refs from Git repository: %s" % unicode(e))
 
@@ -111,7 +124,7 @@
                 logger.info("Requesting commit details for %s" % commit_oids)
             return self._post(
                 "/repo/%s/commits" % path, json={"commits": commit_oids})
-        except Exception as e:
+        except requests.RequestException as e:
             raise GitRepositoryScanFault(
                 "Failed to get commit details from Git repository: %s" %
                 unicode(e))
@@ -127,7 +140,7 @@
             return self._get(
                 "/repo/%s/log/%s" % (path, quote(start)),
                 params={"limit": limit, "stop": stop})
-        except Exception as e:
+        except requests.RequestException as e:
             raise GitRepositoryScanFault(
                 "Failed to get commit log from Git repository: %s" %
                 unicode(e))
@@ -143,7 +156,7 @@
             url = "/repo/%s/compare/%s%s%s" % (
                 path, quote(old), separator, quote(new))
             return self._get(url, params={"context_lines": context_lines})
-        except Exception as e:
+        except requests.RequestException as e:
             raise GitRepositoryScanFault(
                 "Failed to get diff from Git repository: %s" % unicode(e))
 
@@ -157,7 +170,7 @@
             url = "/repo/%s/compare-merge/%s:%s" % (
                 path, quote(base), quote(head))
             return self._get(url, params={"sha1_prerequisite": prerequisite})
-        except Exception as e:
+        except requests.RequestException as e:
             raise GitRepositoryScanFault(
                 "Failed to get merge diff from Git repository: %s" %
                 unicode(e))
@@ -173,7 +186,7 @@
             return self._post(
                 "/repo/%s/detect-merges/%s" % (path, quote(target)),
                 json={"sources": sources})
-        except Exception as e:
+        except requests.RequestException as e:
             raise GitRepositoryScanFault(
                 "Failed to detect merges in Git repository: %s" % unicode(e))
 
@@ -183,7 +196,7 @@
             if logger is not None:
                 logger.info("Deleting repository %s" % path)
             self._delete("/repo/%s" % path)
-        except Exception as e:
+        except requests.RequestException as e:
             raise GitRepositoryDeletionFault(
                 "Failed to delete Git repository: %s" % unicode(e))
 
@@ -195,6 +208,10 @@
                     "Fetching file %s from repository %s" % (filename, path))
             url = "/repo/%s/blob/%s" % (path, quote(filename))
             response = self._get(url, params={"rev": rev})
+        except requests.RequestException as e:
+            raise GitRepositoryScanFault(
+                "Failed to get file from Git repository: %s" % unicode(e))
+        try:
             blob = response["data"].decode("base64")
             if len(blob) != response["size"]:
                 raise GitRepositoryScanFault(

=== modified file 'lib/lp/code/model/tests/test_diff.py'
--- lib/lp/code/model/tests/test_diff.py	2015-10-27 23:35:50 +0000
+++ lib/lp/code/model/tests/test_diff.py	2016-07-01 22:03:52 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2015 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2016 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Tests for Diff, etc."""
@@ -35,11 +35,19 @@
 from lp.services.librarian.interfaces.client import (
     LIBRARIAN_SERVER_DEFAULT_TIMEOUT,
     )
+from lp.services.timeout import (
+    get_default_timeout_function,
+    set_default_timeout_function,
+    )
 from lp.services.webapp import canonical_url
+from lp.services.webapp.adapter import (
+    clear_request_started,
+    set_request_started,
+    )
+from lp.services.webapp.servers import LaunchpadTestRequest
 from lp.testing import (
     login,
     login_person,
-    monkey_patch,
     TestCaseWithFactory,
     verifyObject,
     )
@@ -223,29 +231,44 @@
         diff = self._create_diff(content)
         self.assertTrue(diff.oversized)
 
-    def test_getDiffTimeout(self):
+    def test_timeout(self):
         # The time permitted to get the diff from the librarian may be None,
         # or 2. If there is not 2 seconds left in the request, the number will
-        # will 0.01 smaller or the actual remaining time.
-        content = ''.join(unified_diff('', "1234567890" * 10))
-        diff = self._create_diff(content)
+        # be 0.01 smaller or the actual remaining time.
+        class DiffWithFakeText(Diff):
+            diff_text = FakeMethod()
+
+        diff = DiffWithFakeText()
+        diff.diff_text.open = FakeMethod()
+        diff.diff_text.read = FakeMethod()
+        diff.diff_text.close = FakeMethod()
         value = None
-
-        def fake():
-            return value
-
-        from lp.code.model import diff as diff_module
-        with monkey_patch(diff_module, get_request_remaining_seconds=fake):
-            self.assertIs(
-                LIBRARIAN_SERVER_DEFAULT_TIMEOUT, diff._getDiffTimeout())
-            value = 3.1
-            self.assertEqual(2.0, diff._getDiffTimeout())
-            value = 1.11
-            self.assertEqual(1.1, diff._getDiffTimeout())
-            value = 0.11
-            self.assertEqual(0.1, diff._getDiffTimeout())
-            value = 0.01
-            self.assertEqual(0.01, diff._getDiffTimeout())
+        original_timeout_function = get_default_timeout_function()
+        set_default_timeout_function(lambda: value)
+        try:
+            LaunchpadTestRequest()
+            set_request_started()
+            try:
+                diff.text
+                self.assertEqual(
+                    LIBRARIAN_SERVER_DEFAULT_TIMEOUT,
+                    diff.diff_text.open.calls[-1][0][0])
+                value = 3.1
+                diff.text
+                self.assertEqual(2.0, diff.diff_text.open.calls[-1][0][0])
+                value = 1.11
+                diff.text
+                self.assertEqual(1.1, diff.diff_text.open.calls[-1][0][0])
+                value = 0.11
+                diff.text
+                self.assertEqual(0.1, diff.diff_text.open.calls[-1][0][0])
+                value = 0.01
+                diff.text
+                self.assertEqual(0.01, diff.diff_text.open.calls[-1][0][0])
+            finally:
+                clear_request_started()
+        finally:
+            set_default_timeout_function(original_timeout_function)
 
 
 class TestDiffInScripts(DiffTestCase):

=== modified file 'lib/lp/code/model/tests/test_githosting.py'
--- lib/lp/code/model/tests/test_githosting.py	2016-06-06 11:09:00 +0000
+++ lib/lp/code/model/tests/test_githosting.py	2016-07-01 22:03:52 +0000
@@ -64,13 +64,17 @@
         self.request = None
 
     @contextmanager
-    def mockRequests(self, status_code=200, content=b"",
+    def mockRequests(self, status_code=200, content=b"", reason=None,
                      set_default_timeout=True):
         @all_requests
         def handler(url, request):
             self.assertIsNone(self.request)
             self.request = request
-            return {"status_code": status_code, "content": content}
+            return {
+                "status_code": status_code,
+                "content": content,
+                "reason": reason,
+                }
 
         with HTTMock(handler):
             original_timeout_function = get_default_timeout_function()
@@ -106,10 +110,11 @@
             json_data={"repo_path": "123", "clone_from": "122"})
 
     def test_create_failure(self):
-        with self.mockRequests(status_code=400, content=b"Bad request"):
+        with self.mockRequests(status_code=400, reason=b"Bad request"):
             self.assertRaisesWithContent(
                 GitRepositoryCreationFault,
-                "Failed to create Git repository: Bad request",
+                "Failed to create Git repository: "
+                "400 Client Error: Bad request",
                 self.client.create, "123")
 
     def test_getProperties(self):
@@ -120,10 +125,11 @@
         self.assertRequest("repo/123", method="GET")
 
     def test_getProperties_failure(self):
-        with self.mockRequests(status_code=400, content=b"Bad request"):
+        with self.mockRequests(status_code=400, reason=b"Bad request"):
             self.assertRaisesWithContent(
                 GitRepositoryScanFault,
-                "Failed to get properties of Git repository: Bad request",
+                "Failed to get properties of Git repository: "
+                "400 Client Error: Bad request",
                 self.client.getProperties, "123")
 
     def test_setProperties(self):
@@ -134,10 +140,11 @@
             json_data={"default_branch": "refs/heads/a"})
 
     def test_setProperties_failure(self):
-        with self.mockRequests(status_code=400, content=b"Bad request"):
+        with self.mockRequests(status_code=400, reason=b"Bad request"):
             self.assertRaisesWithContent(
                 GitRepositoryScanFault,
-                "Failed to set properties of Git repository: Bad request",
+                "Failed to set properties of Git repository: "
+                "400 Client Error: Bad request",
                 self.client.setProperties, "123",
                 default_branch="refs/heads/a")
 
@@ -148,10 +155,11 @@
         self.assertRequest("repo/123/refs", method="GET")
 
     def test_getRefs_failure(self):
-        with self.mockRequests(status_code=400, content=b"Bad request"):
+        with self.mockRequests(status_code=400, reason=b"Bad request"):
             self.assertRaisesWithContent(
                 GitRepositoryScanFault,
-                "Failed to get refs from Git repository: Bad request",
+                "Failed to get refs from Git repository: "
+                "400 Client Error: Bad request",
                 self.client.getRefs, "123")
 
     def test_getCommits(self):
@@ -162,11 +170,11 @@
             "repo/123/commits", method="POST", json_data={"commits": ["0"]})
 
     def test_getCommits_failure(self):
-        with self.mockRequests(status_code=400, content=b"Bad request"):
+        with self.mockRequests(status_code=400, reason=b"Bad request"):
             self.assertRaisesWithContent(
                 GitRepositoryScanFault,
                 "Failed to get commit details from Git repository: "
-                "Bad request",
+                "400 Client Error: Bad request",
                 self.client.getCommits, "123", ["0"])
 
     def test_getLog(self):
@@ -185,10 +193,11 @@
             method="GET")
 
     def test_getLog_failure(self):
-        with self.mockRequests(status_code=400, content=b"Bad request"):
+        with self.mockRequests(status_code=400, reason=b"Bad request"):
             self.assertRaisesWithContent(
                 GitRepositoryScanFault,
-                "Failed to get commit log from Git repository: Bad request",
+                "Failed to get commit log from Git repository: "
+                "400 Client Error: Bad request",
                 self.client.getLog, "123", "refs/heads/master")
 
     def test_getDiff(self):
@@ -211,10 +220,11 @@
             "repo/123/compare/a..b?context_lines=4", method="GET")
 
     def test_getDiff_failure(self):
-        with self.mockRequests(status_code=400, content=b"Bad request"):
+        with self.mockRequests(status_code=400, reason=b"Bad request"):
             self.assertRaisesWithContent(
                 GitRepositoryScanFault,
-                "Failed to get diff from Git repository: Bad request",
+                "Failed to get diff from Git repository: "
+                "400 Client Error: Bad request",
                 self.client.getDiff, "123", "a", "b")
 
     def test_getMergeDiff(self):
@@ -242,10 +252,11 @@
         self.assertRequest("repo/123/compare-merge/a:b", method="GET")
 
     def test_getMergeDiff_failure(self):
-        with self.mockRequests(status_code=400, content=b"Bad request"):
+        with self.mockRequests(status_code=400, reason=b"Bad request"):
             self.assertRaisesWithContent(
                 GitRepositoryScanFault,
-                "Failed to get merge diff from Git repository: Bad request",
+                "Failed to get merge diff from Git repository: "
+                "400 Client Error: Bad request",
                 self.client.getMergeDiff, "123", "a", "b")
 
     def test_detectMerges(self):
@@ -257,10 +268,11 @@
             json_data={"sources": ["b", "c"]})
 
     def test_detectMerges_failure(self):
-        with self.mockRequests(status_code=400, content=b"Bad request"):
+        with self.mockRequests(status_code=400, reason=b"Bad request"):
             self.assertRaisesWithContent(
                 GitRepositoryScanFault,
-                "Failed to detect merges in Git repository: Bad request",
+                "Failed to detect merges in Git repository: "
+                "400 Client Error: Bad request",
                 self.client.detectMerges, "123", "a", ["b", "c"])
 
     def test_delete(self):
@@ -269,10 +281,11 @@
         self.assertRequest("repo/123", method="DELETE")
 
     def test_delete_failed(self):
-        with self.mockRequests(status_code=400, content=b"Bad request"):
+        with self.mockRequests(status_code=400, reason=b"Bad request"):
             self.assertRaisesWithContent(
                 GitRepositoryDeletionFault,
-                "Failed to delete Git repository: Bad request",
+                "Failed to delete Git repository: "
+                "400 Client Error: Bad request",
                 self.client.delete, "123")
 
     def test_getBlob(self):
@@ -294,10 +307,11 @@
             "repo/123/blob/dir/path/file/name?rev=dev", method="GET")
 
     def test_getBlob_failure(self):
-        with self.mockRequests(status_code=400, content=b"Bad request"):
+        with self.mockRequests(status_code=400, reason=b"Bad request"):
             self.assertRaisesWithContent(
                 GitRepositoryScanFault,
-                "Failed to get file from Git repository: Bad request",
+                "Failed to get file from Git repository: "
+                "400 Client Error: Bad request",
                 self.client.getBlob, "123", "dir/path/file/name")
 
     def test_getBlob_url_quoting(self):

=== modified file 'lib/lp/services/tests/test_timeout.py'
--- lib/lp/services/tests/test_timeout.py	2016-04-19 12:16:01 +0000
+++ lib/lp/services/tests/test_timeout.py	2016-07-01 22:03:52 +0000
@@ -23,12 +23,18 @@
 
 from lp.services.timeout import (
     get_default_timeout_function,
+    reduced_timeout,
     set_default_timeout_function,
     TimeoutError,
     TransportWithTimeout,
     urlfetch,
     with_timeout,
     )
+from lp.services.webapp.adapter import (
+    clear_request_started,
+    set_request_started,
+    )
+from lp.services.webapp.servers import LaunchpadTestRequest
 from lp.testing import TestCase
 
 
@@ -201,6 +207,36 @@
         no_default_timeout()
         self.assertEqual([True], using_default)
 
+    def test_reduced_timeout(self):
+        """reduced_timeout caps the available timeout in various ways."""
+        self.addCleanup(set_default_timeout_function, None)
+        with reduced_timeout(1.0):
+            self.assertIsNone(get_default_timeout_function()())
+        with reduced_timeout(1.0, default=5.0):
+            self.assertEqual(5.0, get_default_timeout_function()())
+        set_default_timeout_function(lambda: 5.0)
+        with reduced_timeout(1.0):
+            self.assertEqual(4.0, get_default_timeout_function()())
+        with reduced_timeout(1.0, default=5.0, webapp_max=2.0):
+            self.assertEqual(4.0, get_default_timeout_function()())
+        with reduced_timeout(1.0, default=5.0, webapp_max=6.0):
+            self.assertEqual(4.0, get_default_timeout_function()())
+        with reduced_timeout(6.0, default=5.0, webapp_max=2.0):
+            self.assertEqual(5.0, get_default_timeout_function()())
+        LaunchpadTestRequest()
+        set_request_started()
+        try:
+            with reduced_timeout(1.0):
+                self.assertEqual(4.0, get_default_timeout_function()())
+            with reduced_timeout(1.0, default=5.0, webapp_max=2.0):
+                self.assertEqual(2.0, get_default_timeout_function()())
+            with reduced_timeout(1.0, default=5.0, webapp_max=6.0):
+                self.assertEqual(4.0, get_default_timeout_function()())
+            with reduced_timeout(6.0, default=5.0, webapp_max=2.0):
+                self.assertEqual(2.0, get_default_timeout_function()())
+        finally:
+            clear_request_started()
+
     def make_test_socket(self):
         """One common use case for timing out is when making an HTTP request
         to an external site to fetch content. To this end, the timeout

=== modified file 'lib/lp/services/timeout.py'
--- lib/lp/services/timeout.py	2016-04-19 17:18:39 +0000
+++ lib/lp/services/timeout.py	2016-07-01 22:03:52 +0000
@@ -6,6 +6,7 @@
 __metaclass__ = type
 __all__ = [
     "get_default_timeout_function",
+    "reduced_timeout",
     "SafeTransportWithTimeout",
     "set_default_timeout_function",
     "TimeoutError",
@@ -14,6 +15,7 @@
     "with_timeout",
     ]
 
+from contextlib import contextmanager
 import socket
 import sys
 from threading import (
@@ -53,6 +55,42 @@
     default_timeout_function = timeout_function
 
 
+@contextmanager
+def reduced_timeout(clearance, webapp_max=None, default=None):
+    """A context manager that reduces the default timeout.
+
+    :param clearance: The number of seconds by which to reduce the default
+        timeout, to give the call site a chance to recover.
+    :param webapp_max: The maximum permitted time for webapp requests.
+    :param default: The default timeout to use if none is set.
+    """
+    # Circular import.
+    from lp.services.webapp.adapter import get_request_start_time
+
+    original_timeout_function = get_default_timeout_function()
+
+    def timeout():
+        if original_timeout_function is None:
+            return default
+        remaining = original_timeout_function()
+
+        if remaining is None:
+            return default
+        elif (webapp_max is not None and remaining > webapp_max and
+                get_request_start_time() is not None):
+            return webapp_max
+        elif remaining > clearance:
+            return remaining - clearance
+        else:
+            return remaining
+
+    set_default_timeout_function(timeout)
+    try:
+        yield
+    finally:
+        set_default_timeout_function(original_timeout_function)
+
+
 class TimeoutError(Exception):
     """Exception raised when a function doesn't complete within time."""
 

=== modified file 'setup.py'
--- setup.py	2016-05-18 14:54:44 +0000
+++ setup.py	2016-07-01 22:03:52 +0000
@@ -90,6 +90,7 @@
         's4',
         'setproctitle',
         'setuptools',
+        'six',
         'Sphinx',
         'soupmatchers',
         'sourcecodegen',