Merge lp:~pedronis/u1db/u1db-client-http-ops into lp:u1db

[Samuele Pedroni]

support init-db, create, get, put, delete over http in u1db-client,

factor out the logic to open a local or remote db in a common base class, given that we have decided for now that u1db.open shouldn't deal with remote dbs,

for testing convenience, completeness and clarity, factor out the server building bits of u1db-serve to their own functions/file

[Samuele Pedroni (pedronis)]

forgot, in the process added support for a couple more options to u1db-serve (host defaulting to localhost and working dir defaulting to .)

=== modified file 'u1db-serve'
--- u1db-serve	2011-11-22 11:09:00 +0000
+++ u1db-serve	2011-12-01 17:29:35 +0000
@@ -20,9 +20,8 @@
 from u1db import (
     __version__ as _u1db_version,
     )
-from u1db.remote import (
-    http_app,
-    server_state,
+from u1db.commandline import (
+    serve,
     )
 
 
@@ -32,17 +31,16 @@
         description='Run the U1DB server')
     p.add_argument('--version', action='version', version=_u1db_version)
     p.add_argument('--verbose', action='store_true', help='be chatty')
+    p.add_argument('--host', '-H', default='127.0.0.1', metavar='HOST',
+                   help='Bind on this host when serving.')
     p.add_argument('--port', '-p', default=0, metavar='PORT', type=int,
         help='Bind to this port when serving.')
+    p.add_argument('--working-dir', default='.', metavar='WORKING_DIR',
+                   help='Directory where the databases live.')
 
     args = p.parse_args(args)
-
-    state = server_state.ServerState()
-    state.set_workingdir('.')
-    application = http_app.HTTPApp(state)
-    server = httpserver.WSGIServer(application, ('', args.port),
-                                   httpserver.WSGIHandler)
-    sys.stdout.write('listening on port: %s\n' % (server.server_address[1],))
+    server = serve.make_server(args.host, args.port, args.working_dir)
+    sys.stdout.write('listening on: %s:%s\n' % server.server_address)
     sys.stdout.flush()
     server.serve_forever()
 

=== modified file 'u1db/commandline/client.py'
--- u1db/commandline/client.py	2011-12-01 14:15:20 +0000
+++ u1db/commandline/client.py	2011-12-01 17:29:35 +0000
@@ -26,6 +26,7 @@
 from u1db.backends import sqlite_backend
 from u1db.commandline import command
 from u1db.remote import (
+    http_database,
     http_target,
     )
 
@@ -33,14 +34,26 @@
 client_commands = command.CommandGroup()
 
 
-class CmdCreate(command.Command):
+class OneDbCmd(command.Command):
+    """Base class for commands operating on one local or remote database."""
+
+    def _open(self, database, create):
+        if database.startswith('http://'):
+            return http_database.HTTPDatabase.open_database(database, create)
+        else:
+            return u1db_open(database, create)
+
+
+class CmdCreate(OneDbCmd):
     """Create a new document from scratch"""
 
     name = 'create'
 
     @classmethod
     def _populate_subparser(cls, parser):
-        parser.add_argument('database', help='The database to update')
+        parser.add_argument('database',
+                            help='The local or remote database to update',
+                            metavar='database-path-or-url')
         parser.add_argument('infile', nargs='?', default=None,
             help='The file to read content from.')
         parser.add_argument('--id', dest='doc_id', default=None,
@@ -49,27 +62,29 @@
     def run(self, database, infile, doc_id):
         if infile is None:
             infile = self.stdin
-        db = u1db_open(database, create=False)
+        db = self._open(database, create=False)
         doc = db.create_doc(infile.read(), doc_id=doc_id)
         self.stderr.write('id: %s\nrev: %s\n' % (doc.doc_id, doc.rev))
 
 client_commands.register(CmdCreate)
 
 
-class CmdDelete(command.Command):
+class CmdDelete(OneDbCmd):
     """Delete a document from the database"""
 
     name = 'delete'
 
     @classmethod
     def _populate_subparser(cls, parser):
-        parser.add_argument('database', help='The database to update')
+        parser.add_argument('database',
+                            help='The local or remote database to update',
+                            metavar='database-path-or-url')
         parser.add_argument('doc_id', help='The document id to retrieve')
         parser.add_argument('doc_rev',
             help='The revision of the document (which is being superseded.)')
 
     def run(self, database, doc_id, doc_rev):
-        db = u1db_open(database, create=False)
+        db = self._open(database, create=False)
         doc = Document(doc_id, doc_rev, None)
         db.delete_doc(doc)
         self.stderr.write('rev: %s\n' % (doc.rev,))
@@ -77,14 +92,16 @@
 client_commands.register(CmdDelete)
 
 
-class CmdGet(command.Command):
+class CmdGet(OneDbCmd):
     """Extract a document from the database"""
 
     name = 'get'
 
     @classmethod
     def _populate_subparser(cls, parser):
-        parser.add_argument('database', help='The database to query')
+        parser.add_argument('database',
+                            help='The local or remote database to query',
+                            metavar='database-path-or-url')
         parser.add_argument('doc_id', help='The document id to retrieve.')
         parser.add_argument('outfile', nargs='?', default=None,
             help='The file to write the document to',
@@ -93,7 +110,7 @@
     def run(self, database, doc_id, outfile):
         if outfile is None:
             outfile = self.stdout
-        db = u1db_open(database, create=False)
+        db = self._open(database, create=False)
         doc = db.get_doc(doc_id)
         outfile.write(doc.content)
         self.stderr.write('rev: %s\n' % (doc.rev,))
@@ -105,33 +122,37 @@
 client_commands.register(CmdGet)
 
 
-class CmdInitDB(command.Command):
+class CmdInitDB(OneDbCmd):
     """Create a new database"""
 
     name = 'init-db'
 
     @classmethod
     def _populate_subparser(cls, parser):
-        parser.add_argument('database', help='The database to create')
+        parser.add_argument('database',
+                            help='The local or remote database to create',
+                            metavar='database-path-or-url')
         parser.add_argument('--replica-uid', default=None,
-            help='The unique identifier for this database')
+            help='The unique identifier for this database (not for remote)')
 
     def run(self, database, replica_uid):
-        db = u1db_open(database, create=True)
+        db = self._open(database, create=True)
         if replica_uid is not None:
             db._set_replica_uid(replica_uid)
 
 client_commands.register(CmdInitDB)
 
 
-class CmdPut(command.Command):
+class CmdPut(OneDbCmd):
     """Add a document to the database"""
 
     name = 'put'
 
     @classmethod
     def _populate_subparser(cls, parser):
-        parser.add_argument('database', help='The database to update')
+        parser.add_argument('database',
+                            help='The local or remote database to update',
+                            metavar='database-path-or-url'),
         parser.add_argument('doc_id', help='The document id to retrieve')
         parser.add_argument('doc_rev',
             help='The revision of the document (which is being superseded.)')
@@ -142,7 +163,7 @@
     def run(self, database, doc_id, doc_rev, infile):
         if infile is None:
             infile = self.stdin
-        db = u1db_open(database, create=False)
+        db = self._open(database, create=False)
         doc = Document(doc_id, doc_rev, infile.read())
         doc_rev = db.put_doc(doc)
         self.stderr.write('rev: %s\n' % (doc_rev,))

=== added file 'u1db/commandline/serve.py'
--- u1db/commandline/serve.py	1970-01-01 00:00:00 +0000
+++ u1db/commandline/serve.py	2011-12-01 17:29:35 +0000
@@ -0,0 +1,36 @@
+# Copyright 2011 Canonical Ltd.
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 3, as published
+# by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranties of
+# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+# PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Build server for u1db-serve."""
+
+from paste import httpserver
+
+from u1db import (
+    __version__ as _u1db_version,
+    )
+from u1db.remote import (
+    http_app,
+    server_state,
+    )
+
+
+def make_server(host, port, working_dir):
+    """Make a server on host and port exposing dbs living in working_dir."""
+    state = server_state.ServerState()
+    state.set_workingdir(working_dir)
+    application = http_app.HTTPApp(state)
+    server = httpserver.WSGIServer(application, (host, port),
+                                   httpserver.WSGIHandler)
+    return server
+

=== modified file 'u1db/tests/commandline/test_client.py'
--- u1db/tests/commandline/test_client.py	2011-12-01 14:53:45 +0000
+++ u1db/tests/commandline/test_client.py	2011-12-01 17:29:35 +0000
@@ -23,7 +23,10 @@
     open as u1db_open,
     tests,
     )
-from u1db.commandline import client
+from u1db.commandline import (
+    client,
+    serve,
+    )
 from u1db.tests.commandline import safe_close
 from u1db.tests import test_remote_sync_target
 
@@ -287,8 +290,25 @@
         self.assertGetDoc(self.db, doc2.doc_id, doc2.rev, tests.nested_doc,
                           False)
 
-
-class TestCommandLine(TestCaseWithDB):
+class RunMainHelper(object):
+
+    def run_main(self, args, stdin=None):
+        if stdin is not None:
+            self.patch(sys, 'stdin', cStringIO.StringIO(stdin))
+        stdout = cStringIO.StringIO()
+        stderr = cStringIO.StringIO()
+        self.patch(sys, 'stdout', stdout)
+        self.patch(sys, 'stderr', stderr)
+        try:
+            ret = client.main(args)
+        except SystemExit, e:
+            self.fail("Intercepted SystemExit: %s" % (e,))
+        if ret is None:
+            ret = 0
+        return ret, stdout.getvalue(), stderr.getvalue()
+
+
+class TestCommandLine(TestCaseWithDB, RunMainHelper):
     """These are meant to test that the infrastructure is fully connected.
 
     Each command is likely to only have one test here. Something that ensures
@@ -309,21 +329,6 @@
         self.addCleanup(safe_close, p)
         return p
 
-    def run_main(self, args, stdin=None):
-        if stdin is not None:
-            self.patch(sys, 'stdin', cStringIO.StringIO(stdin))
-        stdout = cStringIO.StringIO()
-        stderr = cStringIO.StringIO()
-        self.patch(sys, 'stdout', stdout)
-        self.patch(sys, 'stderr', stderr)
-        try:
-            ret = client.main(args)
-        except SystemExit, e:
-            self.fail("Intercepted SystemExit: %s" % (e,))
-        if ret is None:
-            ret = 0
-        return ret, stdout.getvalue(), stderr.getvalue()
-
     def test_create_subprocess(self):
         p = self.runU1DBClient(['create', '--id', 'test-id', self.db_path])
         stdout, stderr = p.communicate(tests.simple_doc)
@@ -379,3 +384,49 @@
         self.assertEqual('', stdout)
         self.assertEqual('', stderr)
         self.assertGetDoc(self.db2, 'test-id', doc.rev, tests.simple_doc, False)
+
+
+class TestHTTPIntegration(tests.TestCaseWithServer, RunMainHelper):
+    """Meant to test the cases where commands operate over http."""
+
+    def server_def(self):
+        def make_server(host_port, handler, _state):
+            return serve.make_server(host_port[0], host_port[1],
+                                     self.working_dir)
+        return make_server, None, "shutdown", "http"
+
+    def setUp(self):
+        super(TestHTTPIntegration, self).setUp()
+        self.working_dir = self.createTempDir(prefix='u1db-http-server-')
+        self.startServer()
+
+    def getPath(self, dbname):
+        return os.path.join(self.working_dir, dbname)
+
+    def test_init_db(self):
+        url = self.getURL('new.db')
+        ret, stdout, stderr = self.run_main(['init-db', url])
+        db2 = u1db_open(self.getPath('new.db'), create=False)
+
+    def test_create_get_put_delete(self):
+        db = u1db_open(self.getPath('test.db'), create=True)
+        url = self.getURL('test.db')
+        doc_id = '%abcd'
+        ret, stdout, stderr = self.run_main(['create', url, '--id', doc_id],
+                                            stdin=tests.simple_doc)
+        self.assertEqual(0, ret)
+        ret, stdout, stderr = self.run_main(['get', url, doc_id])
+        self.assertEqual(0, ret)
+        self.assertTrue(stderr.startswith('rev: '))
+        doc_rev = stderr[len('rev: '):].rstrip()
+        ret, stdout, stderr = self.run_main(['put', url, doc_id, doc_rev],
+                                            stdin=tests.nested_doc)
+        self.assertEqual(0, ret)
+        self.assertTrue(stderr.startswith('rev: '))
+        doc_rev1 = stderr[len('rev: '):].rstrip()
+        self.assertGetDoc(db, doc_id, doc_rev1, tests.nested_doc, False)
+        ret, stdout, stderr = self.run_main(['delete', url, doc_id, doc_rev1])
+        self.assertEqual(0, ret)
+        self.assertTrue(stderr.startswith('rev: '))
+        doc_rev2 = stderr[len('rev: '):].rstrip()
+        self.assertGetDoc(db, doc_id, doc_rev2, None, False)

=== modified file 'u1db/tests/commandline/test_serve.py'
--- u1db/tests/commandline/test_serve.py	2011-11-23 09:34:59 +0000
+++ u1db/tests/commandline/test_serve.py	2011-12-01 17:29:35 +0000
@@ -19,6 +19,7 @@
 
 from u1db import (
     __version__ as _u1db_version,
+    open as u1db_open,
     tests,
     )
 from u1db.remote import http_client
@@ -49,10 +50,10 @@
 
     def test_bind_to_port(self):
         p = self.startU1DBServe([])
-        starts = 'listening on port:'
+        starts = 'listening on:'
         x = p.stdout.readline()
         self.assertTrue(x.startswith(starts))
-        port = int(x[len(starts):])
+        port = int(x[len(starts):].split(":")[1])
         url = "http://127.0.0.1:%s/" % port
         c = http_client.HTTPClientBase(url)
         self.addCleanup(c.close)
@@ -66,9 +67,31 @@
         s.close()
         p = self.startU1DBServe(['--port', str(port)])
         x = p.stdout.readline().strip()
-        self.assertEqual('listening on port: %s' % (port,), x)
+        self.assertEqual('listening on: 127.0.0.1:%s' % (port,), x)
         url = "http://127.0.0.1:%s/" % port
         c = http_client.HTTPClientBase(url)
         self.addCleanup(c.close)
         res, _ = c._request_json('GET', [])
         self.assertEqual({'version': _u1db_version}, res)
+
+    def test_bind_to_host(self):
+        p = self.startU1DBServe(["--host", "localhost"])
+        starts = 'listening on: 127.0.0.1:'
+        x = p.stdout.readline()
+        self.assertTrue(x.startswith(starts))
+
+    def test_supply_working_dir(self):
+        tmp_dir = self.createTempDir('u1db-serve-test')
+        db = u1db_open(os.path.join(tmp_dir, 'landmark.db'), create=True)
+        db.close()
+        p = self.startU1DBServe(['--working-dir', tmp_dir])
+        starts = 'listening on:'
+        x = p.stdout.readline()
+        self.assertTrue(x.startswith(starts))
+        port = int(x[len(starts):].split(":")[1])
+        url = "http://127.0.0.1:%s/landmark.db" % port
+        c = http_client.HTTPClientBase(url)
+        self.addCleanup(c.close)
+        res, _ = c._request_json('GET', [])
+        self.assertEqual({}, res)
+