Merge lp:~thisfred/desktopcouch/migrate_or_bust into lp:desktopcouch

[Eric Casteleijn]

This adds the migration from 'deleted' flags in records, to moving records into the 'dc_trash' database, where deleted records now should live.

This adds the migration from 'deleted' flags in records, to moving records into the 'dc_trash' database, where deleted records now should live.

[Vincenzo Di Somma (vds)]

review: Approve

[Martin Albisetti (beuno)]

+ """Migrate from deleted falg to the _trash database"""

Typo! You must be so embarrassed....

review: Approve

[Eric Casteleijn (thisfred)]

Fixed!

[Ubuntu One Auto Pilot (otto-pilot)]

The attempt to merge lp:~thisfred/desktopcouch/migrate_or_bust into lp:desktopcouch failed. Below is the output from the failed tests.

Apache CouchDB has started, time to relax.
Browse your desktop CouchDB at file:///tmp/tmpN8Q8Ep/data/couchdb.html
desktopcouch.application.tests.test_start_local_couchdb
  TestUpdateDesignDocuments
    test_create_databases_and_design_docs ... [OK]
desktopcouch.application.tests.test_service
  TestService
    test_start_desktopcouch_replication ... [OK]
    test_start_new_desktopcouch_extensions ... [OK]
    test_start_new_desktopcouch_no_extensions ... [OK]
desktopcouch.application.tests.test_local_files
  TestKeyringIntegration
    test_with_auth ... [OK]
    test_with_no_auth ... [OK]
  TestLocalFiles
    test_all_files_returned ... [OK]
    test_bind_address ... [OK]
    test_couch_chain_ini_files ... [OK]
    test_xdg_overwrite_works ... [OK]
desktopcouch.application.tests.test_replication
  TestReplication
    test_creation ... Apache CouchDB has started, time to relax.
Browse your desktop CouchDB at file:///tmp/tmp2SNWIR/data/couchdb.html
                                                     [OK]
  TestUbuntuoneReplication
    test_exclusion ... Apache CouchDB has started, time to relax.
Browse your desktop CouchDB at file:///tmp/tmpJHrddm/data/couchdb.html
                                                    [OK]
desktopcouch.application.migration.tests.test_migration
  TestMigration
    test_migration_deleted_flag_to_trash ... Traceback (most recent call last):
Failure: testtools.testresult.real._StringException: Text attachment: traceback
------------
Traceback (most recent call last):
  File "/usr/lib/python2.6/dist-packages/testtools/runtest.py", line 144, in _run_user
    return fn(*args)
  File "/usr/lib/python2.6/dist-packages/testtools/testcase.py", line 425, in _run_test_method
    testMethod()
  File "/var/cache/tarmac/desktopcouch/trunk/desktopcouch/application/migration/tests/test_migration.py", line 177, in test_migration_deleted_flag_to_trash
    self.assertEqual(self.dbname, private['original_database_name'])
AssertionError: 'test_migration_deleted_flag_to_trash' != 'dc_trash'
------------

[FAIL]
    test_migration_script_is_run ... [OK]
    test_migration_script_is_run_and_can_access_view ... [OK]
  TestRegistration
    test_register_migration_is_added_to_the_registry ... [OK]
desktopcouch.application.pair.tests.test_couchdb_io
  TestCouchdbIo
    test_get_database_names_replicatable ... [OK]
    test_get_my_host_unique_id ... [OK]
    test_mkuri ... [OK]
    test_obsfuscation ... ...

[Eric Casteleijn (thisfred)]

Oops, forgot to exclude dc_trash from migration.

=== modified file 'bin/desktopcouch-get-port'
--- bin/desktopcouch-get-port	2010-11-22 21:36:06 +0000
+++ bin/desktopcouch-get-port	2010-11-26 19:54:11 +0000
@@ -36,8 +36,6 @@
     # Don't let any of our output interfere
     sys.stdout = DEVNULL
     PORT = desktopcouch.application.platform.find_port()
-    # FIXME log the result of the migration
-    desktopcouch.application.migration.run_needed_migrations()
 finally:
     sys.stdout = SAVED_STDOUT
     DEVNULL.close()

=== modified file 'desktopcouch/application/migration/__init__.py'
--- desktopcouch/application/migration/__init__.py	2010-11-23 18:47:46 +0000
+++ desktopcouch/application/migration/__init__.py	2010-11-26 19:54:11 +0000
@@ -21,64 +21,69 @@
 
 import logging
 
-from couchdb.client import  ResourceConflict
-
 from desktopcouch.records import database
 from desktopcouch.application import server
 from desktopcouch.application.platform import find_port
 
 MIGRATION_DESIGN_DOCUMENT = 'dc_migration'
 
-MIGRATIONS_REGISTRY = {}
-
 
 def _all_dbs(ctx):
-    """Get all the non private dbs from a server"""
+    """Get all the non private dbs from a server."""
     port = find_port(ctx=ctx)
     uri = "http://localhost:%s" % port
     couchdb_server = server.DesktopServer(uri, oauth_tokens=None, ctx=ctx)
-    return [x for x in couchdb_server.resource.get('/_all_dbs')[1]]
-
-
-def _add_view(view_name, view_code, dbs, ctx):
-    """add the provided view to the dbs"""
-    if not dbs:
-        dbs = _all_dbs(ctx)
-    for db_name in [db for db in dbs if not db.startswith('_')]:
-        try:
-            db = server.DesktopDatabase(database=db_name, ctx=ctx)
-        except database.NoSuchDatabase:
-            return
-        try:
-            db.add_view(view_name, map_js=view_code, reduce_js='',
-                        design_doc=MIGRATION_DESIGN_DOCUMENT)
-            logging.info("Migrating DB: %s", db.db.name)
-        except ResourceConflict:
-            pass
-
-
-def register(view_name, view_code, migration_method, ctx, dbs=None):
-    """register a migration script, with the view and the dbs it is meant to
-    migrate"""
+    return [x for x in couchdb_server.resource.get('/_all_dbs')[1] if not
+            x.startswith('_') and not x == database.DCTRASH]
+
+
+def register(migration_method, dbs=None):
+    """Register a migration script, with the view and the dbs it is meant to
+    migrate."""
     if dbs is None:
         dbs = []
-    MIGRATIONS_REGISTRY[view_name] = {'method': migration_method, 'dbs': dbs}
-    _add_view(view_name, view_code, dbs, ctx)
+    MIGRATIONS_REGISTRY.append(
+        {'method': migration_method, 'dbs': dbs})
 
 
 def run_needed_migrations(ctx=None):
-    """Run the actual migration"""
+    """Run the actual migration."""
     from desktopcouch.application import local_files
     if ctx is None:
         ctx = local_files.DEFAULT_CONTEXT
     all_dbs = _all_dbs(ctx)
     for migration in MIGRATIONS_REGISTRY:
-        dbs = MIGRATIONS_REGISTRY[migration]['dbs']
+        dbs = migration['dbs']
         if not dbs:
             dbs = all_dbs
-        for db in dbs:
+        for db_name in dbs:
+            db = server.DesktopDatabase(database=db_name, ctx=ctx)
             try:
-                MIGRATIONS_REGISTRY[migration]['method'](
-                    server.DesktopDatabase(database=db, ctx=ctx))
+                migration['method'](db)
+                logging.info("Migrating DB: %s", db)
             except database.NoSuchDatabase:
                 pass
+
+DELETION_VIEW_NAME = 'migrate_deleted'
+
+DELETION_VIEW_CODE = """
+    function(doc) {
+        if (doc['application_annotations']['Ubuntu One']
+        ['private_application_annotations']['deleted']) {
+        emit(doc.id, doc.id);
+        }
+    }"""
+
+
+def migrate_from_deleted_to_trash(migrateable_db):
+    """Migrate from deleted flag to the dc_trash database."""
+    migrateable_db.add_view(
+        DELETION_VIEW_NAME, map_js=DELETION_VIEW_CODE,
+        design_doc=MIGRATION_DESIGN_DOCUMENT)
+    view_result = migrateable_db.execute_view(
+        view_name=DELETION_VIEW_NAME, design_doc=MIGRATION_DESIGN_DOCUMENT)
+    for result in view_result:
+        migrateable_db.delete_record(result.id)
+
+
+MIGRATIONS_REGISTRY = [{'method': migrate_from_deleted_to_trash, 'dbs': []}]

=== modified file 'desktopcouch/application/migration/tests/test_migration.py'
--- desktopcouch/application/migration/tests/test_migration.py	2010-11-23 18:47:46 +0000
+++ desktopcouch/application/migration/tests/test_migration.py	2010-11-26 19:54:11 +0000
@@ -27,7 +27,8 @@
 from desktopcouch.application import migration
 import desktopcouch.application.tests as test_environment
 from desktopcouch.application.server import DesktopDatabase
-from desktopcouch.records import Record
+from desktopcouch.records import Record, NoRecordTypeSpecified
+from desktopcouch.records.database import DCTRASH
 from desktopcouch.application.stop_local_couchdb import stop_couchdb
 
 
@@ -77,55 +78,25 @@
 
     def test_register_migration_is_added_to_the_registry(self):
         """Test that the migration script is correctly registered."""
-        fake_view_code = ''
         size = len(migration.MIGRATIONS_REGISTRY)
-        migration.register(view_name=self.dbname,
-                           view_code=fake_view_code,
-                           migration_method=fake_migration,
-                           dbs=[self.dbname],
-                           ctx=self.ctx)
-        self.assertEqual(
-            {'method': fake_migration,
-             'dbs': [self.dbname]},
-            migration.MIGRATIONS_REGISTRY[self.dbname])
+        migration.register(migration_method=fake_migration, dbs=[self.dbname])
+        self.assertEqual([{'method': fake_migration, 'dbs': [self.dbname]}],
+            migration.MIGRATIONS_REGISTRY)
         self.assertEqual(size + 1, len(migration.MIGRATIONS_REGISTRY))
 
-    def test_register_migration_add_view_to_a_given_db(self):
-        """Test that the migration is correctly registered."""
-        fake_view_code = ''
-        migration.register(view_name=self.dbname,
-                           view_code=fake_view_code,
-                           migration_method=fake_migration,
-                           dbs=[self.dbname],
-                           ctx=self.ctx)
-        self.assert_(self.database.view_exists(self.dbname, 'dc_migration'))
-        self.failIf(self.one_more_database.view_exists(
-            self.dbname, 'dc_migration'))
-
-    def test_register_migration_add_view_to_all_the_dbs(self):
-        """Test that the migration is correctly registered."""
-        fake_view_code = ''
-        migration.register(view_name=self.dbname,
-                           view_code=fake_view_code,
-                           migration_method=fake_migration,
-                           dbs=[],
-                           ctx=self.ctx)
-        self.assertEqual(
-            {'method': fake_migration,
-             'dbs': []},
-            migration.MIGRATIONS_REGISTRY[self.dbname])
-        for db in [db for db in self.server if not db.startswith('_')]:
-            target = DesktopDatabase(database=db, ctx=self.ctx)
-            self.assert_(target.view_exists(self.dbname, 'dc_migration'))
-
 
 class TestMigration(MigrationBase):
     """Tests the actual migration script"""
 
+    def setUp(self):
+        super(TestMigration, self).setUp()
+        self.trash = DesktopDatabase(DCTRASH, create=True, ctx=self.ctx)
+
     def tearDown(self):
         """tear down each test"""
+        del self.database._server[DCTRASH]
         super(TestMigration, self).tearDown()
-        migration.MIGRATIONS_REGISTRY = {}
+        migration.MIGRATIONS_REGISTRY = []
 
     def test_migration_script_is_run(self):
         """Test that the migration script is run."""
@@ -136,19 +107,20 @@
             # and get the right db as argument
             self.assertEqual(self.dbname, database.db.name)
 
-        fake_view_code = ''
-        migration.register(view_name=self.dbname,
-                           view_code=fake_view_code,
-                           migration_method=simple_migration,
-                           dbs=[self.dbname],
-                           ctx=self.ctx)
+        migration.register(
+            migration_method=simple_migration, dbs=[self.dbname])
         migration.run_needed_migrations(ctx=self.ctx)
 
     def test_migration_script_is_run_and_can_access_view(self):
         """Test that the migration script is run."""
 
+        simple_view_code = 'function(doc){emit(doc._id, doc.record_type)}'
+
         def simple_migration(database):
             """simple migration code"""
+            database.add_view(
+                'simple_view', map_js=simple_view_code,
+                design_doc=migration.MIGRATION_DESIGN_DOCUMENT)
             results = database.execute_view(
                 'simple_view',
                 design_doc='dc_migration')
@@ -161,17 +133,46 @@
         self.database.put_record(Record({
             'key13_1': 'va31_1', 'record_type': 'test.com'}))
 
-        simple_view_code = 'function(doc){emit(doc._id, doc.record_type)}'
-        migration.register(view_name='simple_view',
-                           view_code=simple_view_code,
-                           migration_method=simple_migration,
-                           dbs=[self.dbname],
-                           ctx=self.ctx)
-        migration.run_needed_migrations(ctx=self.ctx)
-        migration.MIGRATIONS_REGISTRY = {}
-        migration.register(view_name='simple_view',
-                           view_code=simple_view_code,
-                           migration_method=simple_migration,
-                           dbs=['db_that_doesnt_exixts'],
-                           ctx=self.ctx)
-        self.assertIs(None, migration.run_needed_migrations(ctx=self.ctx))
+        migration.register(
+            migration_method=simple_migration, dbs=[self.dbname])
+        migration.run_needed_migrations(ctx=self.ctx)
+
+    def test_migration_deleted_flag_to_trash(self):
+        """Test that the migration script is run."""
+        record_id = self.database.put_record(
+            Record({
+                'key1_1': 'val1_1',
+                'record_type': 'test.com',
+                'application_annotations': {
+                    'Ubuntu One': {
+                        'private_application_annotations': {
+                            'deleted': True}}}}))
+        undeleted_record_id1 = self.database.put_record(
+            Record({
+                'key1_1': 'val1_1',
+                'record_type': 'test.com',
+                'application_annotations': {
+                    'Ubuntu One': {
+                        'private_application_annotations': {
+                            'deleted': False}}}}))
+        undeleted_record_id2 = self.database.put_record(
+            Record({
+                'key1_1': 'val1_1',
+                'record_type': 'test.com'}))
+        migration.run_needed_migrations(ctx=self.ctx)
+        # Record no longer exists in database
+        self.assertIs(None, self.database.get_record(record_id))
+        # Undeleted records still exist
+        self.assertIsNot(None, self.database.get_record(undeleted_record_id1))
+        self.assertIsNot(None, self.database.get_record(undeleted_record_id2))
+
+        # Known deleted record is only record in trash
+        for document_id in self.trash.db:
+            try:
+                record = self.trash.get_record(document_id)
+            except NoRecordTypeSpecified:
+                continue
+            private = record.application_annotations['desktopcouch'][
+                'private_application_annotations']
+            self.assertEqual(self.dbname, private['original_database_name'])
+            self.assertEqual(record_id, private['original_id'])

=== modified file 'desktopcouch/application/platform/linux/__init__.py'
--- desktopcouch/application/platform/linux/__init__.py	2010-11-23 18:47:46 +0000
+++ desktopcouch/application/platform/linux/__init__.py	2010-11-26 19:54:11 +0000
@@ -78,6 +78,7 @@
             # now load the design documents and pair records updates,
             # because it's started
             start_local_couchdb.update_design_documents()
+            start_local_couchdb.run_needed_migrations()
             if not process_is_couchdb(pid):
                 logging.error("CouchDB process did not start up")
                 raise RuntimeError("desktop-couch not started")

=== modified file 'desktopcouch/application/start_local_couchdb.py'
--- desktopcouch/application/start_local_couchdb.py	2010-11-24 14:40:03 +0000
+++ desktopcouch/application/start_local_couchdb.py	2010-11-26 19:54:11 +0000
@@ -46,7 +46,7 @@
 import sys
 import time
 
-from desktopcouch.application import local_files
+from desktopcouch.application import local_files, migration
 from desktopcouch.application.platform import (
     read_pidfile, process_is_couchdb, find_port)
 from desktopcouch.application.server import DesktopDatabase
@@ -118,7 +118,6 @@
     return pid, port
 
 
-# pylint: disable=R0914
 def update_design_documents(ctx=local_files.DEFAULT_CONTEXT):
     """Check system design documents and update any that need updating
 
@@ -176,7 +175,11 @@
                 else:
                     db.add_view(
                         view_name, mapjs, design_doc=dd_name)
-# pylint: enable=R0914
+
+
+def run_needed_migrations(ctx=local_files.DEFAULT_CONTEXT):
+    """Run any necessary migrations."""
+    migration.run_needed_migrations(ctx=ctx)
 
 
 def update_bookmark_file(port, ctx=local_files.DEFAULT_CONTEXT):

=== modified file 'desktopcouch/records/database.py'
--- desktopcouch/records/database.py	2010-11-24 19:31:39 +0000
+++ desktopcouch/records/database.py	2010-11-26 19:54:11 +0000
@@ -219,7 +219,7 @@
         return self.db.query(map_fun, reduce_fun, language,
                              wrapper, **options)
 
-    def get_record(self, record_id, hide_deleted=True):
+    def get_record(self, record_id):
         """Get a record from back end storage."""
 
         def make_getter(source_db, document_id, attachment_name, content_type):
@@ -239,8 +239,6 @@
             return None
         data = {}
 
-        if row_is_deleted(couch_record) and hide_deleted:
-            return None
         data.update(couch_record)
         record = self.record_factory(data=data)
         if "_attachments" in data: