Merge lp:~pedronis/u1db/http-app-sync-testing into lp:u1db

[Samuele Pedroni]

Integration testing for the sync parts of http-app and a start of a HTTP client side, for now it just introduces a HTTPSyncTarget (but subsequently we will need a full HTTPDatabase).

todo for later branches: error handling checking response status

[lp:~pedronis/u1db/http-app-sync-testing]

119. By Samuele Pedroni on 2011-11-16

merge trunk

[John A Meinel (jameinel)]

def prepare(dic):

looks like it would be easier to use as:

def prepare(**dic):

Since the two callers use prepare(dict(...)) they could just call prepare(...)
(minor thing, though)

I like that it runs against both servers, though that may not last particularly long. What I really like, though, is that as a transition step, we know that both work the same way.

I'm curious why you call the HTTPConnection a _client:
43 + self._client = httplib.HTTPConnection(self._url.hostname,
44 + self._url.port)

review: Approve

[Samuele Pedroni (pedronis)]

> def prepare(dic):
>
> looks like it would be easier to use as:
>
> def prepare(**dic):
>
> Since the two callers use prepare(dict(...)) they could just call prepare(...)
> (minor thing, though)

true

>
> I like that it runs against both servers, though that may not last
> particularly long. What I really like, though, is that as a transition step,
> we know that both work the same way.
>

yup

>
>
> I'm curious why you call the HTTPConnection a _client:
> 43 + self._client = httplib.HTTPConnection(self._url.hostname,
> 44 + self._url.port)

was just easier to share tests with that name, can be renamed later when the tests are organized differently

[lp:~pedronis/u1db/http-app-sync-testing]

120. By Samuele Pedroni on 2011-11-17

small simplification

=== added file 'u1db/remote/http_target.py'
--- u1db/remote/http_target.py	1970-01-01 00:00:00 +0000
+++ u1db/remote/http_target.py	2011-11-17 14:38:24 +0000
@@ -0,0 +1,93 @@
+# Copyright 2011 Canonical Ltd.
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 3, as published
+# by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranties of
+# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+# PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+""""""
+
+import httplib
+import json
+import urlparse
+
+from u1db import (
+    SyncTarget,
+    )
+
+class HTTPSyncTarget(SyncTarget):
+    """Implement the SyncTarget api to a remote HTTP server."""
+
+    @staticmethod
+    def connect(url):
+        return HTTPSyncTarget(url)
+
+    def __init__(self, url):
+        self._url = urlparse.urlsplit(url)
+        self._client = None
+
+    def _ensure_connection(self):
+        if self._client is not None:
+            return
+        self._client = httplib.HTTPConnection(self._url.hostname,
+                                              self._url.port)
+
+    def get_sync_info(self, other_replica_uid):
+        self._ensure_connection()
+        self._client.request('GET', '%s/sync-from/%s' % (self._url.path,
+                                                         other_replica_uid))
+        # xxx check for errors with status
+        res = json.loads(self._client.getresponse().read())
+        return (res['this_replica_uid'], res['this_replica_generation'],
+                res['other_replica_generation'])
+
+    def record_sync_info(self, other_replica_uid, other_replica_generation):
+        self._ensure_connection()
+        self._client.request('PUT',
+                          '%s/sync-from/%s' % (self._url.path,
+                                               other_replica_uid),
+                          json.dumps({'generation': other_replica_generation}),
+                          {'content-type': 'application/json'})
+        self._client.getresponse().read() # xxx check for errors with status
+
+    def sync_exchange(self, docs_info, from_replica_uid,
+                      from_replica_generation,
+                      last_known_generation, return_doc_cb):
+        self._ensure_connection()
+        self._client.putrequest('POST',
+                                '%s/sync-from/%s' % (self._url.path,
+                                                     from_replica_uid))
+        self._client.putheader('content-type', 'application/x-u1db-multi-json')
+        entries = []
+        size = 0
+        def prepare(**dic):
+            entry = json.dumps(dic)+"\r\n"
+            entries.append(entry)
+            return len(entry)
+        size += prepare(last_known_generation=last_known_generation,
+                        from_replica_generation=from_replica_generation)
+        for doc_id, doc_rev, doc in docs_info:
+            size += prepare(id=doc_id, rev=doc_rev, doc=doc)
+        self._client.putheader('content-length', str(size))
+        self._client.endheaders()
+        for entry in entries:
+            self._client.send(entry)
+        entries = None
+        resp = self._client.getresponse() # xxx check for errors with status
+        data = resp.read().splitlines() # one at a time
+        res = json.loads(data[0])
+        for entry in data[1:]:
+            entry = json.loads(entry)
+            return_doc_cb(entry['id'], entry['rev'], entry['doc'])
+        data = None
+        return res['new_generation']
+
+    def get_sync_exchange(self):
+        return None # not a local target

=== modified file 'u1db/tests/__init__.py'
--- u1db/tests/__init__.py	2011-11-04 14:46:39 +0000
+++ u1db/tests/__init__.py	2011-11-17 14:38:24 +0000
@@ -124,28 +124,40 @@
         self.sent_response = True
 
 
-class TestCaseWithSyncServer(TestCase):
+class TestCaseWithServer(TestCase):
+
+    @staticmethod
+    def server_def():
+        # should return (ServerClass, RequestHandlerClass,
+        #                "shutdown method name", "url_scheme")
+        raise NotImplementedError(TestCaseWithServer.server_def)
 
     def setUp(self):
-        super(TestCaseWithSyncServer, self).setUp()
+        super(TestCaseWithServer, self).setUp()
         self.server = self.server_thread = None
 
-    def startServer(self, request_handler=sync_server.TCPSyncRequestHandler):
+    @property
+    def url_scheme(self):
+        return self.server_def()[-1]
+
+    def startServer(self, other_request_handler=None):
+        server_def = self.server_def()
+        server_class, request_handler, shutdown_meth, _ = server_def
+        request_handler = other_request_handler or request_handler
         self.request_state = ServerStateForTests()
-        self.server = sync_server.TCPSyncServer(
-            ('127.0.0.1', 0), request_handler,
-            self.request_state)
+        self.server = server_class(('127.0.0.1', 0), request_handler,
+                                   self.request_state)
         self.server_thread = threading.Thread(target=self.server.serve_forever,
                                               kwargs=dict(poll_interval=0.01))
         self.server_thread.start()
         self.addCleanup(self.server_thread.join)
-        self.addCleanup(self.server.force_shutdown)
+        self.addCleanup(getattr(self.server, shutdown_meth))
 
     def getURL(self, path=None):
         host, port = self.server.server_address
         if path is None:
             path = ''
-        return 'u1db://%s:%s/%s' % (host, port, path)
+        return '%s://%s:%s/%s' % (self.url_scheme, host, port, path)
 
 
 def socket_pair():

=== modified file 'u1db/tests/commandline/test_client.py'
--- u1db/tests/commandline/test_client.py	2011-11-14 13:52:12 +0000
+++ u1db/tests/commandline/test_client.py	2011-11-17 14:38:24 +0000
@@ -27,6 +27,7 @@
     )
 from u1db.commandline import client
 from u1db.tests.commandline import safe_close
+from u1db.tests import test_remote_sync_server
 
 
 class TestArgs(tests.TestCase):
@@ -192,7 +193,11 @@
                          self.db.get_doc('my-test-id'))
 
 
-class TestCmdSyncRemote(tests.TestCaseWithSyncServer, TestCaseWithDB):
+class TestCmdSyncRemote(tests.TestCaseWithServer, TestCaseWithDB):
+
+    @staticmethod
+    def server_def():
+        return test_remote_sync_server.TestTCPSyncServer.server_def()
 
     def setUp(self):
         super(TestCmdSyncRemote, self).setUp()

=== modified file 'u1db/tests/test_remote_sync_server.py'
--- u1db/tests/test_remote_sync_server.py	2011-11-01 19:07:07 +0000
+++ u1db/tests/test_remote_sync_server.py	2011-11-17 14:38:24 +0000
@@ -51,15 +51,12 @@
         self.request.sendall('goodbye\n')
 
 
-class TestTestCaseWithSyncServer(tests.TestCaseWithSyncServer):
-
-    def test_getURL(self):
-        self.startServer()
-        url = self.getURL()
-        self.assertTrue(url.startswith('u1db://127.0.0.1:'))
-
-
-class TestTCPSyncServer(tests.TestCaseWithSyncServer):
+class TestTCPSyncServer(tests.TestCaseWithServer):
+
+    @staticmethod
+    def server_def():
+        return (sync_server.TCPSyncServer, sync_server.TCPSyncRequestHandler,
+                "force_shutdown", "u1db")
 
     def connectToServer(self):
         client_sock = socket.socket()

=== modified file 'u1db/tests/test_remote_sync_target.py'
--- u1db/tests/test_remote_sync_target.py	2011-11-02 22:42:05 +0000
+++ u1db/tests/test_remote_sync_target.py	2011-11-17 14:38:24 +0000
@@ -12,38 +12,72 @@
 # You should have received a copy of the GNU General Public License along
 # with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-"""Tests for the RemoteSyncTarget"""
+"""Tests for the remote sync targets"""
 
 import os
+from wsgiref import simple_server
+#from paste import httpserver
 
 from u1db import (
     tests,
     )
 from u1db.remote import (
+    sync_server,
     sync_target,
+    http_app,
+    http_target
     )
 from u1db.backends import (
     sqlite_backend,
     )
 
 
-class TestRemoteSyncTarget(tests.TestCaseWithSyncServer):
+def remote_server_def():
+    return (sync_server.TCPSyncServer, sync_server.TCPSyncRequestHandler,
+            "force_shutdown", "u1db")
+
+def http_server_def():
+    def make_server(host_port, handler, state):
+        application = http_app.HTTPApp(state)
+        srv = simple_server.WSGIServer(host_port, handler)
+        srv.set_app(application)
+        #srv = httpserver.WSGIServerBase(application,
+        #                                host_port,
+        #                                handler
+        #                                )
+        return srv
+    class req_handler(simple_server.WSGIRequestHandler):
+        def log_request(*args):
+            pass # suppress
+    #rh = httpserver.WSGIHandler
+    return make_server, req_handler, "shutdown", "http"
+
+
+class TestRemoteSyncTargets(tests.TestCaseWithServer):
+
+    scenarios = [
+        ('http', {'server_def': http_server_def,
+                  'sync_target_class': http_target.HTTPSyncTarget}),
+        ('remote', {'server_def': remote_server_def,
+                    'sync_target_class': sync_target.RemoteSyncTarget}),
+        ]
 
     def getSyncTarget(self, path=None):
         if self.server is None:
             self.startServer()
-        return sync_target.RemoteSyncTarget.connect(self.getURL(path))
+        return self.sync_target_class(self.getURL(path))
 
     def test_connect(self):
         self.startServer()
         url = self.getURL()
-        remote_target = sync_target.RemoteSyncTarget.connect(url)
+        remote_target = self.sync_target_class.connect(url)
         self.assertEqual(url, remote_target._url.geturl())
-        self.assertIs(None, remote_target._conn)
+        self.assertIs(None, remote_target._client)
 
     def test_parse_url(self):
-        remote_target = sync_target.RemoteSyncTarget('u1db://127.0.0.1:12345/')
-        self.assertEqual('u1db', remote_target._url.scheme)
+        remote_target = self.sync_target_class(
+                                     '%s://127.0.0.1:12345/' % self.url_scheme)
+        self.assertEqual(self.url_scheme, remote_target._url.scheme)
         self.assertEqual('127.0.0.1', remote_target._url.hostname)
         self.assertEqual(12345, remote_target._url.port)
         self.assertEqual('/', remote_target._url.path)
@@ -54,13 +88,12 @@
 
     def test__ensure_connection(self):
         remote_target = self.getSyncTarget()
-        self.assertIs(None, remote_target._conn)
-        remote_target._ensure_connection()
-        self.assertIsNot(None, remote_target._conn)
-        c = remote_target._conn
-        remote_target._ensure_connection()
-        self.assertIs(c, remote_target._conn)
+        self.assertIs(None, remote_target._client)
+        remote_target._ensure_connection()
         self.assertIsNot(None, remote_target._client)
+        cli = remote_target._client
+        remote_target._ensure_connection()
+        self.assertIs(cli, remote_target._client)
 
     def test_get_sync_info(self):
         self.startServer()
@@ -106,3 +139,5 @@
                         last_known_generation=0, return_doc_cb=receive_doc)
         self.assertEqual(1, new_gen)
         self.assertEqual([(doc_id, doc_rev, {'value': 'there'})], other_docs)
+
+load_tests = tests.load_with_scenarios

=== modified file 'u1db/tests/test_sync.py'
--- u1db/tests/test_sync.py	2011-11-02 22:42:05 +0000
+++ u1db/tests/test_sync.py	2011-11-17 14:38:24 +0000
@@ -14,6 +14,7 @@
 
 """The Synchronization class for U1DB."""
 
+from wsgiref import simple_server
 
 from u1db import (
     errors,
@@ -23,8 +24,14 @@
     )
 from u1db.remote import (
     sync_target,
+    http_app,
+    http_target,
     )
 
+from u1db.tests.test_remote_sync_target import (
+    http_server_def,
+    remote_server_def,
+    )
 
 simple_doc = tests.simple_doc
 nested_doc = tests.nested_doc
@@ -43,14 +50,24 @@
     return db, st
 
 
+def _make_local_db_and_http_target(test):
+    test.startServer()
+    db = test.request_state._create_database('test')
+    st = http_target.HTTPSyncTarget.connect(test.getURL('test'))
+    return db, st
+
+
 target_scenarios = [
     ('local', {'create_db_and_target': _make_local_db_and_target}),
-    ('remote', {'create_db_and_target': _make_local_db_and_remote_target}),
+    ('remote', {'create_db_and_target': _make_local_db_and_remote_target,
+                'server_def': remote_server_def}),
+    ('http', {'create_db_and_target': _make_local_db_and_http_target,
+              'server_def': http_server_def}),
     ]
 
 
 class DatabaseSyncTargetTests(tests.DatabaseBaseTests,
-                              tests.TestCaseWithSyncServer):
+                              tests.TestCaseWithServer):
 
     scenarios = tests.multiply_scenarios(tests.DatabaseBaseTests.scenarios,
                                          target_scenarios)

=== added file 'u1db/tests/test_test_infrastructure.py'
--- u1db/tests/test_test_infrastructure.py	1970-01-01 00:00:00 +0000
+++ u1db/tests/test_test_infrastructure.py	2011-11-17 14:38:24 +0000
@@ -0,0 +1,36 @@
+# Copyright 2011 Canonical Ltd.
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 3, as published
+# by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranties of
+# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+# PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Tests for test infrastructure bits"""
+
+from wsgiref import simple_server
+
+from u1db import (
+    tests,
+    )
+
+
+class TestTestCaseWithServer(tests.TestCaseWithServer):
+
+    @staticmethod
+    def server_def():
+        def make_server(host_port, handler, state):
+            return simple_server.WSGIServer(host_port, handler)
+        return (make_server, simple_server.WSGIRequestHandler,
+                "shutdown", "http")
+
+    def test_getURL(self):
+        self.startServer()
+        url = self.getURL()
+        self.assertTrue(url.startswith('http://127.0.0.1:'))