Merge lp:~alecu/ubuntuone-storage-protocol/timestamp-autofix-1-4 into lp:ubuntuone-storage-protocol/stable-1-4

[Alejandro J. Cura]

Do a HEAD request on the server to get accurate timestamp (LP: #692597)

Do a HEAD request on the server to get accurate timestamp (LP: #692597)

[Diego Sarmentero (diegosarmentero)]

Text conflict in tests/test_client.py
Text conflict in ubuntuone/storageprotocol/client.py
2 conflicts encountered.

review: Needs Fixing

[Diego Sarmentero (diegosarmentero)]

review: Abstain

[Diego Sarmentero (diegosarmentero)]

+1

review: Approve

[Natalia Bidart (nataliabidart)]

Looks good, tested IRL.

review: Approve

[Ubuntu One Auto Pilot (otto-pilot)]

The attempt to merge lp:~alecu/ubuntuone-storage-protocol/timestamp-autofix-1-4 into lp:ubuntuone-storage-protocol/stable-1-4 failed. Below is the output from the failed tests.

running build
running build_py
creating build
creating build/lib.linux-i686-2.6
creating build/lib.linux-i686-2.6/ubuntuone
copying ubuntuone/__init__.py -> build/lib.linux-i686-2.6/ubuntuone
creating build/lib.linux-i686-2.6/ubuntuone/storageprotocol
copying ubuntuone/storageprotocol/context.py -> build/lib.linux-i686-2.6/ubuntuone/storageprotocol
copying ubuntuone/storageprotocol/proxy_tunnel.py -> build/lib.linux-i686-2.6/ubuntuone/storageprotocol
copying ubuntuone/storageprotocol/content_hash.py -> build/lib.linux-i686-2.6/ubuntuone/storageprotocol
copying ubuntuone/storageprotocol/protocol_pb2.py -> build/lib.linux-i686-2.6/ubuntuone/storageprotocol
copying ubuntuone/storageprotocol/__init__.py -> build/lib.linux-i686-2.6/ubuntuone/storageprotocol
copying ubuntuone/storageprotocol/utils.py -> build/lib.linux-i686-2.6/ubuntuone/storageprotocol
copying ubuntuone/storageprotocol/volumes.py -> build/lib.linux-i686-2.6/ubuntuone/storageprotocol
copying ubuntuone/storageprotocol/delta.py -> build/lib.linux-i686-2.6/ubuntuone/storageprotocol
copying ubuntuone/storageprotocol/errors.py -> build/lib.linux-i686-2.6/ubuntuone/storageprotocol
copying ubuntuone/storageprotocol/client.py -> build/lib.linux-i686-2.6/ubuntuone/storageprotocol
copying ubuntuone/storageprotocol/dircontent.py -> build/lib.linux-i686-2.6/ubuntuone/storageprotocol
copying ubuntuone/storageprotocol/dircontent_pb2.py -> build/lib.linux-i686-2.6/ubuntuone/storageprotocol
copying ubuntuone/storageprotocol/validators.py -> build/lib.linux-i686-2.6/ubuntuone/storageprotocol
copying ubuntuone/storageprotocol/sharersp.py -> build/lib.linux-i686-2.6/ubuntuone/storageprotocol
copying ubuntuone/storageprotocol/request.py -> build/lib.linux-i686-2.6/ubuntuone/storageprotocol
running check

error: No such file or directory

=== modified file 'tests/test_client.py'
--- tests/test_client.py	2010-07-26 17:15:04 +0000
+++ tests/test_client.py	2011-11-23 19:22:23 +0000
@@ -1,8 +1,9 @@
 # -*- coding: utf-8 -*-
 #
 # Author: Natalia B. Bidart <[e-mail]>
+# Author: Alejandro J. Cura <[e-mail]>
 #
-# Copyright (C) 2009 Canonical Ltd.
+# Copyright (C) 2009, 2011 Canonical Ltd.
 #
 # This program is free software: you can redistribute it and/or modify it
 # under the terms of the GNU Affero General Public License version 3,
@@ -17,15 +18,21 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """Tests for the protocol client."""
 
+import time
 import unittest
 import uuid
 
-from twisted.internet.defer import Deferred
+from twisted.application import internet, service
+from twisted.internet import defer
+from twisted.internet.defer import Deferred, inlineCallbacks
+from twisted.trial.unittest import TestCase as TwistedTestCase
+from twisted.web import server, resource
 
 from ubuntuone.storageprotocol import protocol_pb2, sharersp, delta, request
 from ubuntuone.storageprotocol.client import (
     StorageClient, CreateUDF, ListVolumes, DeleteVolume, GetDelta,
-    Authenticate, MakeFile, MakeDir, PutContent, Move, Unlink
+    Authenticate, MakeFile, MakeDir, PutContent, Move, Unlink,
+    oauth, TwistedTimestampChecker, tx_timestamp_checker,
 )
 from ubuntuone.storageprotocol import volumes
 from tests import test_delta_info
@@ -703,7 +710,7 @@
         self.assertEquals(self.request.share_id,
                           actual_msg.get_delta.share)
 
-class TestAuth(unittest.TestCase):
+class TestAuth(TwistedTestCase):
     """Tests the authentication request."""
 
     def test_session_id(self):
@@ -719,6 +726,140 @@
         self.assert_(request.done.called)
         self.assertEqual(request.session_id, SESSION_ID)
 
+    def test_oauth_authenticate_uses_server_timestamp(self):
+        """The oauth authentication uses the server timestamp."""
+        fromcandt_call = []
+
+        fake_token = oauth.OAuthToken('token', 'token_secret')
+        fake_consumer = oauth.OAuthConsumer('consumer_key', 'consumer_secret')
+
+        fake_timestamp = object()
+        timestamp_d = Deferred()
+        self.patch(tx_timestamp_checker, "get_faithful_time",
+                   lambda: timestamp_d)
+        original_fromcandt = oauth.OAuthRequest.from_consumer_and_token
+
+        @staticmethod
+        def fake_from_consumer_and_token(**kwargs):
+            """A fake from_consumer_and_token."""
+            fromcandt_call.append(kwargs)
+            return original_fromcandt(**kwargs)
+
+        self.patch(oauth.OAuthRequest, "from_consumer_and_token",
+                   fake_from_consumer_and_token)
+        protocol = FakedProtocol()
+        protocol.oauth_authenticate(fake_consumer, fake_token)
+        self.assertEqual(len(fromcandt_call), 0)
+        timestamp_d.callback(fake_timestamp)
+        parameters = fromcandt_call[0]["parameters"]
+        self.assertEqual(parameters["oauth_timestamp"], fake_timestamp)
+
+
+class RootResource(resource.Resource):
+    """A root resource that logs the number of calls."""
+
+    isLeaf = True
+
+    def __init__(self, *args, **kwargs):
+        """Initialize this fake instance."""
+        self.count = 0
+        self.request_headers = []
+
+    def render_HEAD(self, request):
+        """Increase the counter on each render."""
+        self.request_headers.append(request.requestHeaders)
+        self.count += 1
+        return ""
+
+
+class MockWebServer(object):
+    """A mock webserver for testing."""
+
+    def __init__(self):
+        """Start up this instance."""
+        self.root = RootResource()
+        site = server.Site(self.root)
+        application = service.Application('web')
+        self.service_collection = service.IServiceCollection(application)
+        self.tcpserver = internet.TCPServer(0, site)
+        self.tcpserver.setServiceParent(self.service_collection)
+        self.service_collection.startService()
+
+    def get_url(self):
+        """Build the url for this mock server."""
+        port_num = self.tcpserver._port.getHost().port
+        return "http://localhost:%d/" % port_num
+
+    def stop(self):
+        """Shut it down."""
+        self.service_collection.stopService()
+
+
+class TimestampCheckerTestCase(TwistedTestCase):
+    """Tests for the timestamp checker."""
+
+    def setUp(self):
+        """Initialize a fake webserver."""
+        self.ws = MockWebServer()
+        self.addCleanup(self.ws.stop)
+        self.patch(TwistedTimestampChecker, "SERVER_URL", self.ws.get_url())
+
+    @inlineCallbacks
+    def test_returned_value_is_int(self):
+        """The returned value is an integer."""
+        checker = TwistedTimestampChecker()
+        t = yield checker.get_faithful_time()
+        self.assertEqual(type(t), int)
+
+    @inlineCallbacks
+    def test_first_call_does_head(self):
+        """The first call gets the clock from our web."""
+        checker = TwistedTimestampChecker()
+        yield checker.get_faithful_time()
+        self.assertEqual(self.ws.root.count, 1)
+
+    @inlineCallbacks
+    def test_second_call_is_cached(self):
+        """For the second call, the time is cached."""
+        checker = TwistedTimestampChecker()
+        yield checker.get_faithful_time()
+        yield checker.get_faithful_time()
+        self.assertEqual(self.ws.root.count, 1)
+
+    @inlineCallbacks
+    def test_after_timeout_cache_expires(self):
+        """After some time, the cache expires."""
+        fake_timestamp = 1
+        self.patch(time, "time", lambda: fake_timestamp)
+        checker = TwistedTimestampChecker()
+        yield checker.get_faithful_time()
+        fake_timestamp += TwistedTimestampChecker.CHECKING_INTERVAL
+        yield checker.get_faithful_time()
+        self.assertEqual(self.ws.root.count, 2)
+
+    @inlineCallbacks
+    def test_server_error_means_skew_not_updated(self):
+        """When server can't be reached, the skew is not updated."""
+        fake_timestamp = 1
+        self.patch(time, "time", lambda: fake_timestamp)
+        checker = TwistedTimestampChecker()
+        failing_get_server_time = lambda: defer.fail(FakedError())
+        self.patch(checker, "get_server_time", failing_get_server_time)
+        yield checker.get_faithful_time()
+        self.assertEqual(checker.skew, 0)
+        self.assertEqual(checker.next_check,
+                    fake_timestamp + TwistedTimestampChecker.ERROR_INTERVAL)
+
+    @inlineCallbacks
+    def test_server_date_sends_nocache_headers(self):
+        """Getting the server date sends the no-cache headers."""
+        checker = TwistedTimestampChecker()
+        yield checker.get_server_date_header(self.ws.get_url())
+        assert len(self.ws.root.request_headers) == 1
+        headers = self.ws.root.request_headers[0]
+        result = headers.getRawHeaders("Cache-Control")
+        self.assertEqual(result, ["no-cache"])
+
 
 class TestGenerationInRequests(unittest.TestCase):
     """Base class for testing that actions that change the volume will

=== modified file 'ubuntuone/storageprotocol/client.py'
--- ubuntuone/storageprotocol/client.py	2010-07-26 17:15:04 +0000
+++ ubuntuone/storageprotocol/client.py	2011-11-23 19:22:23 +0000
@@ -2,8 +2,9 @@
 #
 # Author: Lucio Torre <[e-mail]>
 # Author: Natalia B. Bidart <[e-mail]>
+# Author: Alejandro J. Cura <[e-mail]>
 #
-# Copyright 2009 Canonical Ltd.
+# Copyright 2009, 2011 Canonical Ltd.
 #
 # This program is free software: you can redistribute it and/or modify it
 # under the terms of the GNU Affero General Public License version 3,
@@ -28,13 +29,35 @@
 from twisted.internet.protocol import ClientFactory
 from twisted.internet import reactor, defer
 from twisted.python import log
+from twisted.web import client
+from twisted.web.http_headers import Headers
 
 from ubuntuone.storageprotocol import (
-    protocol_pb2, request, sharersp, volumes, delta
+    delta,
+    protocol_pb2,
+    request,
+    sharersp,
+    utils,
+    volumes,
 )
 
 log_debug = partial(log.msg, loglevel=logging.DEBUG)
 
+class TwistedTimestampChecker(utils.BaseTimestampChecker):
+    """A timestamp that's regularly checked with a server."""
+
+    @defer.inlineCallbacks
+    def get_server_date_header(self, server_url):
+        """Get the server date using twisted webclient."""
+        agent = client.Agent(reactor)
+        headers = Headers({'Cache-Control': ['no-cache']})
+        response = yield agent.request("HEAD", server_url, headers)
+        defer.returnValue(response.headers.getRawHeaders("Date")[0])
+
+
+tx_timestamp_checker = TwistedTimestampChecker()
+
+
 class StorageClient(request.RequestHandler):
     """A Basic Storage Protocol client."""
 
@@ -98,6 +121,7 @@
         p.start()
         return p.deferred
 
+    @defer.inlineCallbacks
     def oauth_authenticate(self, consumer, token):
         """Authenticate to a server using the OAuth provider.
 
@@ -110,9 +134,11 @@
         object when completed.
 
         """
+        timestamp = yield tx_timestamp_checker.get_faithful_time()
         request = oauth.OAuthRequest.from_consumer_and_token(
             oauth_consumer=consumer,
             token=token,
+            parameters={"oauth_timestamp": timestamp},
             http_method="GET",
             http_url="storage://server")
         request.sign_request(
@@ -123,7 +149,8 @@
             (key, str(value)) for key, value in request.parameters.iteritems())
         p = Authenticate(self, auth_parameters)
         p.start()
-        return p.deferred
+        result = yield p.deferred
+        defer.returnValue(result)
 
     def handle_ROOT(self, message):
         """Handle incoming ROOT message.

=== added file 'ubuntuone/storageprotocol/utils.py'
--- ubuntuone/storageprotocol/utils.py	1970-01-01 00:00:00 +0000
+++ ubuntuone/storageprotocol/utils.py	2011-11-23 19:22:23 +0000
@@ -0,0 +1,70 @@
+# ubuntuone.storageprotocol.utils - some storage protocol utils
+#
+# Author: Alejandro J. Cura <[e-mail]>
+#
+# Copyright 2011 Canonical Ltd.
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License version 3,
+# as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranties of
+# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+# PURPOSE.  See the GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""Some storage protocol utils."""
+
+import logging
+import time
+
+from functools import partial
+
+from twisted.internet import defer
+from twisted.python import log
+from twisted.web import http
+
+log_debug = partial(log.msg, loglevel=logging.DEBUG)
+
+
+class BaseTimestampChecker(object):
+    """A timestamp that's regularly checked with a server."""
+
+    CHECKING_INTERVAL = 60 * 60  # in seconds
+    ERROR_INTERVAL = 30  # in seconds
+    SERVER_URL = "http://one.ubuntu.com/api/time"
+
+    def __init__(self):
+        """Initialize this instance."""
+        self.next_check = time.time()
+        self.skew = 0
+
+    def get_server_date_header(self, server_url):
+        """Return a deferred with the server time, using your web client."""
+        return defer.fail(NotImplementedError())
+
+    @defer.inlineCallbacks
+    def get_server_time(self):
+        """Get the time at the server."""
+        date_string = yield self.get_server_date_header(self.SERVER_URL)
+        timestamp = http.stringToDatetime(date_string)
+        defer.returnValue(timestamp)
+
+    @defer.inlineCallbacks
+    def get_faithful_time(self):
+        """Get an accurate timestamp."""
+        local_time = time.time()
+        if local_time >= self.next_check:
+            try:
+                server_time = yield self.get_server_time()
+                self.next_check = local_time + self.CHECKING_INTERVAL
+                self.skew = server_time - local_time
+                log_debug("Calculated server-local time skew:", self.skew)
+            except Exception, e:
+                log_debug("Error while verifying the server time skew:", e)
+                self.next_check = local_time + self.ERROR_INTERVAL
+        log_debug("Using corrected timestamp:",
+                  http.datetimeToString(local_time + self.skew))
+        defer.returnValue(int(local_time + self.skew))