Merge lp:~vds/desktopcouch/simple_migration_infrastructure into lp:desktopcouch

[Vincenzo Di Somma]

Add basic migration infrastructure, to facilitate at least a planned switch from deleted-flagged records to a separate Trash database. (LP: #670700, #675590)

Basic migration infrastructure.

[lp:~vds/desktopcouch/simple_migration_infrastructure]

211. By Vincenzo Di Somma on 2010-11-16

pylint happy (again)

[Eric Casteleijn (thisfred)]

Looks good, tests pass.

review: Approve

[lp:~vds/desktopcouch/simple_migration_infrastructure]

212. By Vincenzo Di Somma on 2010-11-16

thisfred happy

[Manuel de la Peña (mandel)]

This is a +1 from me. Nevertheless we should aim to get all our unittests to use mocker to mock the db access. I have added a bug report and assign it to vds so that we do not forget, bug number is lp:676476

review: Approve

[Chad Miller (cmiller)]

review: Approve

=== modified file 'bin/desktopcouch-get-port'
--- bin/desktopcouch-get-port	2010-02-25 15:50:50 +0000
+++ bin/desktopcouch-get-port	2010-11-16 18:24:15 +0000
@@ -16,6 +16,9 @@
 # along with desktopcouch.  If not, see <http://www.gnu.org/licenses/>.
 #
 # Authors: Chad Miller <[e-mail]>
+#          Eric Casteleijn <[e-mail]>
+#          Vincenzo Di Somma <[e-mail]>
+
 
 """Return on stdout the port that the personal couchdb is listening on.
 As always, success returns a zero exit code, and an exception in Python
@@ -32,6 +35,8 @@
     # Don't let any of our output interfere
     sys.stdout = devnull
     port = desktopcouch.find_port()
+    # FIXME log the result of the migration
+    desktopcouch.migration.run_needed_migrations()
 finally:
     sys.stdout = saved_stdout
     devnull.close()

=== added directory 'desktopcouch/migration'
=== added file 'desktopcouch/migration/__init__.py'
--- desktopcouch/migration/__init__.py	1970-01-01 00:00:00 +0000
+++ desktopcouch/migration/__init__.py	2010-11-16 18:24:15 +0000
@@ -0,0 +1,80 @@
+# Copyright 2010 Canonical Ltd.
+#
+# This file is part of desktopcouch.
+#
+# desktopcouch is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3
+# as published by the Free Software Foundation.
+#
+# desktopcouch is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with desktopcouch.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors: Eric Casteleijn <[e-mail]>
+#          Vincenzo Di Somma <[e-mail]>
+
+"""Basic migration infrastructure"""
+
+import logging
+
+from couchdb.client import  ResourceConflict
+
+from desktopcouch.records import server, server_base
+from desktopcouch.platform import find_port
+
+MIGRATION_DESIGN_DOCUMENT = 'dc_migration'
+
+MIGRATIONS_REGISTRY = {}
+
+def _all_dbs(ctx):
+    """Get all the non private dbs from a server"""
+    port = find_port(ctx=ctx)
+    uri = "http://localhost:%s" % port
+    couchdb_server = server.OAuthCapableServer(
+        uri, oauth_tokens=None, ctx=ctx)
+    return [x for x in couchdb_server.resource.get('/_all_dbs')[1]]
+        
+def _add_view(view_name, view_code, dbs, ctx):
+    """add the provided view to the dbs"""
+    if not dbs:
+        dbs = _all_dbs(ctx)
+    for db_name in [db for db in dbs if not db.startswith('_')]:
+        try:
+            db = server.CouchDatabase(database=db_name, ctx=ctx)
+        except server_base.NoSuchDatabase:
+            return
+        try:
+            db.add_view(view_name, map_js=view_code, reduce_js='',
+                        design_doc=MIGRATION_DESIGN_DOCUMENT)
+            logging.info("Migrating DB: ", db.db.name)
+        except ResourceConflict:
+            pass
+
+def register(view_name, view_code, migration_method, ctx, dbs=None):
+    """register a migration script, with the view and the dbs it is meant to
+    migrate"""
+    if dbs is None:
+        dbs = []
+    MIGRATIONS_REGISTRY[view_name] = {'method': migration_method, 'dbs': dbs}
+    _add_view(view_name, view_code, dbs, ctx)
+
+def run_needed_migrations(ctx=None):
+    """Run the actual migration"""
+    from desktopcouch import local_files
+    if ctx is None:
+        ctx = local_files.DEFAULT_CONTEXT
+    all_dbs = _all_dbs(ctx)
+    for migration in MIGRATIONS_REGISTRY:
+        dbs = MIGRATIONS_REGISTRY[migration]['dbs']
+        if not dbs:
+            dbs = all_dbs
+        for db in dbs:
+            try:
+                MIGRATIONS_REGISTRY[migration]['method'](
+                    server.CouchDatabase(database=db, ctx=ctx))
+            except server_base.NoSuchDatabase:
+                pass

=== added directory 'desktopcouch/migration/tests'
=== added file 'desktopcouch/migration/tests/__init__.py'
--- desktopcouch/migration/tests/__init__.py	1970-01-01 00:00:00 +0000
+++ desktopcouch/migration/tests/__init__.py	2010-11-16 18:24:15 +0000
@@ -0,0 +1,1 @@
+"""Tests suites for migration infrastructure"""

=== added file 'desktopcouch/migration/tests/test_migration.py'
--- desktopcouch/migration/tests/test_migration.py	1970-01-01 00:00:00 +0000
+++ desktopcouch/migration/tests/test_migration.py	2010-11-16 18:24:15 +0000
@@ -0,0 +1,176 @@
+# Copyright 2010 Canonical Ltd.
+#
+# This file is part of desktopcouch.
+#
+# desktopcouch is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3
+# as published by the Free Software Foundation.
+#
+# desktopcouch is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with desktopcouch.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors: Eric Casteleijn <[e-mail]>
+#          Vincenzo Di Somma <[e-mail]>
+
+"""Tests for migration infrastructure"""
+
+#pylint: disable=W0212
+#tests can access private attributes
+
+import testtools
+
+from desktopcouch import migration
+import desktopcouch.tests as test_environment
+from desktopcouch.records.server import CouchDatabase
+from desktopcouch.records.record import Record
+from desktopcouch.stop_local_couchdb import stop_couchdb
+
+
+def get_test_context():
+    """Return test context."""
+    return test_environment.test_context
+
+def fake_migration():
+    """fake migration code"""
+    pass
+
+
+class MigrationBase(testtools.TestCase):
+    """Tests the registration of a migration script"""
+
+    def setUp(self):
+        """setup each test"""
+        super(MigrationBase, self).setUp()
+        self.ctx = get_test_context()
+        self.dbname = self._testMethodName
+        self.database = CouchDatabase(self.dbname, create=True, ctx=self.ctx)
+        self.server = self.database._server
+
+    def tearDown(self):
+        """tear down each test"""
+        del self.database._server[self.database._database_name]
+        this_context = get_test_context()
+        if this_context != test_environment.test_context:
+            stop_couchdb(ctx=this_context)
+        super(MigrationBase, self).tearDown()
+
+class TestRegistration(MigrationBase):
+    """Tests the registration of a migration script"""
+
+    def setUp(self):
+        """setup each test"""
+        super(TestRegistration, self).setUp()
+        self.one_more_database = CouchDatabase('one_more_database',
+                                               create=True,
+                                               ctx=self.ctx)
+
+    def tearDown(self):
+        """tear down each test"""
+        del self.database._server[self.one_more_database._database_name]
+        super(TestRegistration, self).tearDown()
+
+    def test_register_migration_is_added_to_the_registry(self):
+        """Test that the migration script is correctly registered."""
+        fake_view_code = ''
+        size = len(migration.MIGRATIONS_REGISTRY)
+        migration.register(view_name=self.dbname,
+                           view_code=fake_view_code,
+                           migration_method=fake_migration,
+                           dbs=[self.dbname],
+                           ctx=self.ctx)
+        self.assertEqual(
+            {'method': fake_migration,
+             'dbs':[self.dbname]},
+            migration.MIGRATIONS_REGISTRY[self.dbname])
+        self.assertEqual(size+1, len(migration.MIGRATIONS_REGISTRY))
+        
+    def test_register_migration_add_view_to_a_given_db(self):
+        """Test that the migration is correctly registered."""
+        fake_view_code = ''
+        migration.register(view_name=self.dbname,
+                           view_code=fake_view_code,
+                           migration_method=fake_migration,
+                           dbs=[self.dbname],
+                           ctx=self.ctx)
+        self.assert_(self.database.view_exists(self.dbname, 'dc_migration'))
+        self.failIf(self.one_more_database.view_exists(
+            self.dbname, 'dc_migration'))
+
+    def test_register_migration_add_view_to_all_the_dbs(self):
+        """Test that the migration is correctly registered."""
+        fake_view_code = ''
+        migration.register(view_name=self.dbname,
+                           view_code=fake_view_code,
+                           migration_method=fake_migration,
+                           dbs=[],
+                           ctx=self.ctx)
+        self.assertEqual(
+            {'method': fake_migration,
+             'dbs':[]},
+            migration.MIGRATIONS_REGISTRY[self.dbname])
+        for db in [db for db in self.server if not db.startswith('_')]:
+            target = CouchDatabase(database=db, ctx=self.ctx)
+            self.assert_(target.view_exists(self.dbname, 'dc_migration'))
+
+
+class TestMigration(MigrationBase):
+    """Tests the actual migration script"""
+
+    def tearDown(self):
+        """tear down each test"""
+        super(TestMigration, self).tearDown()
+        migration.MIGRATIONS_REGISTRY = {}
+    def test_migration_script_is_run(self):
+        """Test that the migration script is run."""
+
+        def simple_migration(database):
+            """simple migration code"""
+            # just asserting something to check this code is run
+            # and get the right db as argument
+            self.assertEqual(self.dbname, database.db.name)
+
+        fake_view_code = ''
+        migration.register(view_name=self.dbname,
+                           view_code=fake_view_code,
+                           migration_method=simple_migration,
+                           dbs=[self.dbname],
+                           ctx=self.ctx)
+        migration.run_needed_migrations(ctx=self.ctx)
+        
+    def test_migration_script_is_run_and_can_access_view(self):
+        """Test that the migration script is run."""
+
+        def simple_migration(database):
+            """simple migration code"""
+            results = database.execute_view(
+                'simple_view',
+                design_doc='dc_migration')
+            self.assertEqual(3, len(results))
+
+        self.database.put_record(Record({
+            'key1_1': 'val1_1', 'record_type': 'test.com'}))
+        self.database.put_record(Record({
+            'key2_1': 'val2_1', 'record_type': 'test.com'}))
+        self.database.put_record(Record({
+            'key13_1': 'va31_1', 'record_type': 'test.com'}))
+
+        simple_view_code = 'function(doc){emit(doc._id, doc.record_type)}'
+        migration.register(view_name='simple_view',
+                           view_code=simple_view_code,
+                           migration_method=simple_migration,
+                           dbs=[self.dbname],
+                           ctx=self.ctx)
+        migration.run_needed_migrations(ctx=self.ctx)
+        migration.MIGRATIONS_REGISTRY = {}
+        migration.register(view_name='simple_view',
+                           view_code=simple_view_code,
+                           migration_method=simple_migration,
+                           dbs=['db_that_doesnt_exixts'],
+                           ctx=self.ctx)
+        self.assertIs(None, migration.run_needed_migrations(ctx=self.ctx))
+

=== modified file 'runtests.py'
--- runtests.py	2010-11-12 11:56:35 +0000
+++ runtests.py	2010-11-16 18:24:15 +0000
@@ -42,6 +42,6 @@
     coverage.stop()
     subprocess.call(["utilities/lint.sh", ""], shell=True)
     source_files = ffind('desktopcouch')
-    coverage.report(source_files, ignore_errors=1, show_missing=0)
+    coverage.report(source_files, ignore_errors=1, show_missing=1)
     sys.exit(not return_code)
     # pylint: enable-msg=C0103