Merge lp:~abentley/charmworld/config-storage into lp:~juju-jitsu/charmworld/trunk

[Aaron Bentley]

Change the internal representation of config options, so that the user-supplied option names are used as values, not as keys.

[lp:~abentley/charmworld/config-storage]

296. By Aaron Bentley on 2013-06-27

Merged trunk into config-storage.

[Curtis Hovey (sinzui)]

We talked on IRC to clarify that some charms are not in a sane state and migration needs to handle cases where config is not a dict with a single item of key, sub-dict. We might want migrate to log in the future. We know from the 007 migration problem seen with production's bogus charms that the juju-log is capturing stdout.

review: Approve (code)

[lp:~abentley/charmworld/config-storage]

297. By Aaron Bentley on 2013-07-04

Merged slow-migrations into config-storage.

298. By Aaron Bentley on 2013-07-04

Fix test failures by adding error translation to delete_charm.

299. By Aaron Bentley on 2013-07-04

Merged slow-migrations into config-storage.

300. By Aaron Bentley on 2013-07-04

Fix failing test.

301. By Aaron Bentley on 2013-07-04

Convert options migration to an exodus.

302. By Aaron Bentley on 2013-07-04

Merged slow-migrations into config-storage.

303. By Aaron Bentley on 2013-07-04

Remove unused code.

304. By Aaron Bentley on 2013-07-05

Merged slow-migrations-2 into config-storage.

305. By Aaron Bentley on 2013-07-05

Merged slow-migrations-2 into config-storage.

Unmerged revisions

=== modified file 'charmworld/jobs/ingest.py'
--- charmworld/jobs/ingest.py	2013-07-01 14:45:02 +0000
+++ charmworld/jobs/ingest.py	2013-07-05 14:15:30 +0000
@@ -32,11 +32,14 @@
     get_branch_info,
     parse_date,
 )
-from charmworld.models import getconnection
-from charmworld.models import getdb
-from charmworld.models import getfs
-from charmworld.models import CharmFileSet
-from charmworld.models import CharmSource
+from charmworld.models import (
+    CharmFileSet,
+    CharmSource,
+    getconnection,
+    getdb,
+    getfs,
+    options_to_storage,
+)
 from charmworld.search import (
     ElasticSearchClient,
     SearchServiceNotAvailable,
@@ -491,7 +494,11 @@
         config_raw = cfile.read()
 
         try:
-            config = quote_yaml(yaml.load(config_raw))
+            config_yaml = yaml.load(config_raw)
+            if 'options' in config_yaml:
+                config_yaml['options'] = options_to_storage(
+                    config_yaml['options'])
+            config = quote_yaml(config_yaml)
         except Exception, exc:
             raise IngestError(
                 'Invalid charm config yaml. %s: %s' % (
 
=== modified file 'charmworld/jobs/tests/test_cstat.py'
--- charmworld/jobs/tests/test_cstat.py	2013-07-01 15:57:20 +0000
+++ charmworld/jobs/tests/test_cstat.py	2013-07-05 14:15:30 +0000
@@ -33,7 +33,8 @@
             def send(request):
                 response = Response()
                 if ('start' in request.url):
-                    response._content = '[["2010-12-24", 4], ["2010-12-25", 1]]'
+                    response._content = ('[["2010-12-24", 4],'
+                                         ' ["2010-12-25", 1]]')
                 else:
                     response._content = '[[25]]'
                 return response
 
=== modified file 'charmworld/jobs/tests/test_scan.py'
--- charmworld/jobs/tests/test_scan.py	2013-06-24 15:45:29 +0000
+++ charmworld/jobs/tests/test_scan.py	2013-07-05 14:15:30 +0000
@@ -34,8 +34,40 @@
         # 'foo.bar' is replaced with 'foo%2Ebar'.
         self.assertFalse('dotted.name' in self.charm)
         self.assertEqual('foo', self.charm['dotted%2Ename'])
-        self.assertFalse('foo.bar' in self.charm['config']['options'])
-        self.assertEqual('baz', self.charm['config']['options']['foo%2Ebar'])
+
+    def test_options_format(self):
+        del self.charm['config']
+        scan_charm(self.charm, self.db, self.fs, self.log)
+        expected = [
+            {
+                'name': 'source',
+                'default': 'lp:charmworld',
+                'type': 'string',
+                'description': 'The bzr branch to pull the charmworld source'
+                               ' from.'
+            },
+            {
+                'name': 'revno',
+                'default': -1,
+                'type': 'int',
+                'description': 'The revno of the bzr branch to use. -1 for'
+                               ' current tip.'
+            },
+            {
+                'name': 'execute-ingest-every',
+                'default': 15,
+                'type': 'int',
+                'description': 'The amount of time (in minutes) that the'
+                               ' ingest tasks for charmworld are executed'
+            },
+            {
+                'name': 'error-email',
+                'default': '',
+                'type': 'string',
+                'description': 'Address to mail errors to.'
+            }
+        ]
+        self.assertItemsEqual(expected, self.charm['config']['options'])
 
     def test_short_interface_specification(self):
         # Charms can specify the interfaces they require or provide
 
=== modified file 'charmworld/models.py'
--- charmworld/models.py	2013-07-01 14:45:02 +0000
+++ charmworld/models.py	2013-07-05 14:15:30 +0000
@@ -544,7 +544,11 @@
 
         The dict is parsed from the config_raw property's YAML.
         """
-        return self._representation['config']
+        config = self._representation['config']
+        if config is not None and 'options' in config:
+            config = dict(config)
+            config['options'] = storage_to_options(config['options'])
+        return config
 
     @property
     def config_raw(self):
@@ -557,7 +561,7 @@
     @property
     def options(self):
         """The charm's configuration options."""
-        config = self._representation['config']
+        config = self.config
         if config is not None and 'options' in config:
             return config['options']
         else:
@@ -969,8 +973,10 @@
 
 class CharmSource:
 
-    def __init__(self, db, index_client):
-        self.db = db
+    def __init__(self, db, index_client, collection=None):
+        if collection is None:
+            collection = db.charms
+        self.collection = collection
         self.index_client = index_client
 
     @classmethod
@@ -984,16 +990,20 @@
         return cls(request.db, request.index_client)
 
     def sync_index(self):
-        for charm in self.db.charms.find():
+        for charm in self.collection.find():
             yield charm['_id']
             self.index_client.index_charm(charm)
 
     def save(self, charm):
-        self.db.charms.save(charm)
-        self.index_client.index_charm(charm)
+        self.collection.save(charm)
+        try:
+            self.index_client.index_charm(charm)
+        except:
+            self.index_client.delete_charm(charm['_id'])
+            raise
 
     def get_store_charms(self):
-        return self.db.charms.find({'store_url': {'$exists': True}})
+        return self.collection.find({'store_url': {'$exists': True}})
 
     @staticmethod
     def exclude_obsolete(query):
@@ -1002,7 +1012,7 @@
     def find_mia_charms(self):
         query = self.exclude_obsolete(
             {"store_data.errors": {"$exists": True}})
-        return (Charm(data) for data in self.db.charms.find(query))
+        return (Charm(data) for data in self.collection.find(query))
 
     def find_proof_error_charms(self, sub_only):
         query = {"proof.e": {"$exists": True}, "promulgated": True}
@@ -1010,13 +1020,16 @@
             query['subordinate'] = True
         query = self.exclude_obsolete(query)
         sort = [("series", pymongo.DESCENDING), ("name", pymongo.ASCENDING)]
-        return (Charm(data) for data in self.db.charms.find(query, sort=sort))
+        return (Charm(data) for data in self.collection.find(query, sort=sort))
 
     def _get_all(self, charm_id):
         """"Return all stored values for a given charm."""
-        yield self.db.charms.find_one({'_id': charm_id})
+        yield self.collection.find_one({'_id': charm_id})
         yield self.index_client.get(charm_id)
 
+    def create_collection(self):
+        self.collection.database.create_collection(self.collection.name)
+
 
 def acquire_session_secret(database):
     """Return a secret to use for AuthTkt sessions.
@@ -1073,3 +1086,33 @@
             logger.exception('Unable to index charm.')
         else:
             logger.info('Syncing %s to Elasticsearch' % charm_id)
+
+
+def options_to_storage(options):
+    """Convert options, as retrieved from config.yaml, into a list.
+
+    This format avoids user-defined keys, which are problematic in
+    elasticsearch.
+    """
+    storage = []
+    for key, value in options.items():
+        option = {'name': key}
+        option.update(value)
+        storage.append(option)
+    return storage
+
+
+def storage_to_options(storage):
+    """Convert options, as retrieved from mongodb, into a dict.
+
+    This format has the same structure as config.yaml.
+    """
+    options = {}
+    for option in storage:
+        new_option = dict(option)
+        del new_option['name']
+        value = options.setdefault(option['name'], new_option)
+        if value is not new_option:
+            raise ValueError('Option name "%s" occurs multiple times.' %
+                             option['name'])
+    return options
 
=== modified file 'charmworld/routes.py'
--- charmworld/routes.py	2013-07-02 12:28:28 +0000
+++ charmworld/routes.py	2013-07-05 14:15:30 +0000
@@ -89,7 +89,8 @@
     # up and running. They may need to be consulted when this is changed to a
     # later API version.
     config.add_route('api_latest_single', '/api/latest/{endpoint}')
-    config.add_view('charmworld.views.api.API2', route_name='api_latest_single')
+    config.add_view('charmworld.views.api.API2',
+                    route_name='api_latest_single')
     config.add_route('api_latest', '/api/latest/{endpoint}/*remainder')
     config.add_view('charmworld.views.api.API2', route_name='api_latest')
 
 
=== modified file 'charmworld/search.py'
--- charmworld/search.py	2013-07-01 14:45:02 +0000
+++ charmworld/search.py	2013-07-05 14:15:30 +0000
@@ -30,7 +30,7 @@
     'name': 10,
     'summary': 5,
     'description': 3,
-    'config.options.*.description': None,
+    'config.options.description': None,
     'relations': None,
 }
 
@@ -100,6 +100,14 @@
         self._client = client
         self.index_name = index_name
 
+    def exists(self):
+        try:
+            self.get_status()
+        except IndexMissing:
+            return False
+        else:
+            return True
+
     def create_index(self):
         self._client.create_index(self.index_name)
 
@@ -146,13 +154,24 @@
         """
         exact_index = ['categories', 'name', 'owner', 'i_provides',
                        'i_requires', 'series', 'store_url']
+        charm_properties = dict(
+            (name, {'type': 'string', 'index': 'not_analyzed'})
+            for name in exact_index)
+        charm_properties['config'] = {
+            'properties': {
+                'options': {
+                    'properties': {
+                        'default': {'type': 'string'},
+                        'description': {'type': 'string'},
+                    }
+                }
+            }
+        }
+
         with translate_error():
             self._client.put_mapping(self.index_name, 'charm', {
                 'charm': {
-                    'properties': dict(
-                        (name, {'type': 'string', 'index': 'not_analyzed'})
-                        for name in exact_index
-                    )
+                    'properties': charm_properties,
                 }
             })
 
@@ -166,6 +185,10 @@
             self._client.index(self.index_name, 'charm', charm, charm_id,
                                refresh=True)
 
+    def delete_charm(self, charm_id):
+        with translate_error():
+            self._client.delete(self.index_name, 'charm', charm_id)
+
     def index_charms(self, charms):
         if len(charms) == 0:
             return
@@ -311,6 +334,10 @@
         hits = self._unlimited_search(dsl, limit)
         return [hit['_source'] for hit in hits['hits']['hits']]
 
+    def get_status(self):
+        with translate_error():
+            return self._client.status(index=self.index_name)['indices']
+
     def search(self, terms, valid_charms_only=True):
         # Avoid circular imports.
         from charmworld.models import Charm
@@ -323,8 +350,7 @@
         dsl = self._get_filtered(dsl, {}, None, valid_charms_only)
 
         with Timer() as timer:
-            with translate_error():
-                status = self._client.status(index=self.index_name)['indices']
+            status = self.get_status()
             for real_index in self.get_aliased():
                 break
             else:
@@ -395,6 +421,9 @@
                 result_r.setdefault(interface, []).append(payload)
         return result_r, result_p,
 
+    def get_related_client(self, name):
+        return self.__class__(self._client, name)
+
     def get_aliasable_client(self, name=None):
         """Return a client that can be aliased to this one.
 
@@ -403,7 +432,18 @@
         """
         if name is None:
             name = '%s-%d' % (self.index_name, random.randint(1, 99999))
-        return self.__class__(self._client, name)
+        return self.get_related_client(name)
+
+    @contextmanager
+    def replacement(self, name=None):
+        copy = self.get_aliasable_client(name)
+        copy.create_index()
+        try:
+            copy.put_mapping()
+            yield copy
+        except:
+            copy.delete_index()
+            raise
 
     def create_replacement(self, name=None, charms=None):
         """Create a replacement for this index.
@@ -417,10 +457,7 @@
             index will be used.
         :return: The ElasticSearchClient for the supplied mapping.
         """
-        copy = self.get_aliasable_client(name)
-        copy.create_index()
-        try:
-            copy.put_mapping()
+        with self.replacement(name) as copy:
             if charms is None:
                 try:
                     charms = self.api_search(valid_charms_only=False)
@@ -428,9 +465,6 @@
                     charms = []
             copy.index_charms(charms)
             return copy
-        except:
-            copy.delete_index()
-            raise
 
 
 def reindex(index_client, charms=None):
 
=== modified file 'charmworld/testing/__init__.py'
--- charmworld/testing/__init__.py	2013-06-24 15:08:43 +0000
+++ charmworld/testing/__init__.py	2013-07-05 14:15:30 +0000
@@ -60,9 +60,9 @@
         self.addCleanup(context.__exit__, None, None, None)
         return result
 
-    def use_index_client(self, put_mapping=True):
+    def use_index_client(self, put_mapping=True, aliased=False):
         self.index_client = self.use_context(
-            temp_index_client(put_mapping=put_mapping))
+            temp_index_client(put_mapping=put_mapping, aliased=aliased))
         return self.index_client
 
     @contextmanager
@@ -223,13 +223,18 @@
 
 
 @contextmanager
-def temp_index_client(name='temp_index', put_mapping=True):
+def temp_index_client(name='temp_index', put_mapping=True, aliased=False):
     client = ElasticSearchClient.from_settings(INI, name)
     try:
         client.delete_index()
     except IndexMissing:
         pass
-    client.create_index()
+    if aliased:
+        alias_client = client.get_aliasable_client()
+        alias_client.create_index()
+        client.update_aliased(alias_client.index_name, [])
+    else:
+        client.create_index()
     try:
         if put_mapping:
             client.put_mapping()
 
=== modified file 'charmworld/testing/data/sample_charm/config.yaml'
--- charmworld/testing/data/sample_charm/config.yaml	2013-02-07 17:26:59 +0000
+++ charmworld/testing/data/sample_charm/config.yaml	2013-07-05 14:15:30 +0000
@@ -15,4 +15,3 @@
     type: string
     default: ""
     description: "Address to mail errors to."
-  foo.bar: baz
 
=== modified file 'charmworld/testing/factory.py'
--- charmworld/testing/factory.py	2013-07-01 14:45:02 +0000
+++ charmworld/testing/factory.py	2013-07-05 14:15:30 +0000
@@ -24,8 +24,9 @@
     process_charm,
 )
 from charmworld.models import (
+    CharmFileSet,
     getfs,
-    CharmFileSet,
+    options_to_storage,
 )
 
 
@@ -185,6 +186,7 @@
                 'description': 'The interval between script runs.'
             },
         }
+    options = options_to_storage(options)
     if files is None:
         files = {
             'readme': {
 
=== modified file 'charmworld/tests/test_models.py'
--- charmworld/tests/test_models.py	2013-07-01 14:45:02 +0000
+++ charmworld/tests/test_models.py	2013-07-05 14:15:30 +0000
@@ -14,6 +14,10 @@
 from os.path import join
 
 from mock import patch
+from pyelasticsearch.exceptions import (
+    ElasticHttpError,
+    ElasticHttpNotFoundError,
+)
 
 from charmworld.models import (
     acquire_session_secret,
@@ -22,7 +26,9 @@
     CharmFileSet,
     CharmSource,
     find_charms,
+    options_to_storage,
     QAData,
+    storage_to_options,
     sync_index,
     UserMgr,
 )
@@ -437,11 +443,10 @@
 
     def test_config(self):
         # The config property is a dict of charm options.
-        config = {'options': {'mode': 'fast'}}
-        charm_data = factory.get_charm_json()
-        charm_data['config'] = config
+        config = {'options': {'key': {'default': 'value', 'type': 'string'}}}
+        charm_data = factory.get_charm_json(options=config['options'])
         charm = Charm(charm_data)
-        self.assertIs(config, charm.config)
+        self.assertEqual(config, charm.config)
         # The default is a dict with a single key named 'options' set to
         # an empty dict.
         charm = Charm({})
@@ -460,12 +465,10 @@
 
     def test_options(self):
         # The options property is a dict from the charm's config.
-        options = {'mode': 'fast'}
-        config = {'options': options}
-        charm_data = factory.get_charm_json()
-        charm_data['config'] = config
+        options = {'key': {'default': 'value', 'type': 'string'}}
+        charm_data = factory.get_charm_json(options=options)
         charm = Charm(charm_data)
-        self.assertIs(options, charm.options)
+        self.assertEqual(options, charm.options)
         # The default is a dict.
         charm = Charm({})
         self.assertEqual({}, charm.options)
@@ -1040,3 +1043,58 @@
             messages = [r.getMessage() for r in handler.buffer]
             self.assertIn('Unable to index charm.', messages)
         self.assertEqual(1, len(self.index_client.api_search()))
+
+    def test_save_deletes_on_error(self):
+        self.use_index_client()
+        source = CharmSource.from_request(self)
+        source.save({'_id': 'a', 'b': {}})
+        with self.assertRaises(ElasticHttpError):
+            source.save({'_id': 'a', 'b': 'c'})
+        with self.assertRaises(ElasticHttpNotFoundError):
+            self.index_client.get('a')
+
+
+class TestOptionsStorage(TestCase):
+
+    yaml = {
+        'foo': {
+            'default': 'bar',
+            'type': 'string',
+            'description': 'Hello.'
+        },
+        'baz': {
+            'default': 42,
+            'type': 'int',
+            'description': 'Hello.'
+        },
+    }
+    storage = [{
+        'name': 'baz',
+        'default': 42,
+        'type': 'int',
+        'description': 'Hello.'
+    }, {
+        'name': 'foo',
+        'default': 'bar',
+        'type': 'string',
+        'description': 'Hello.'
+    }]
+
+    def test_options_to_storage(self):
+        self.assertItemsEqual(self.storage, options_to_storage(self.yaml))
+
+    def test_storage_to_options(self):
+        self.assertEqual(self.yaml, storage_to_options(self.storage))
+
+    def test_storage_to_options_duplicate_key(self):
+        with self.assertRaises(ValueError) as exc_context:
+            storage_to_options([{'name': 'same'}, {'name': 'same'}])
+        self.assertEqual('Option name "same" occurs multiple times.',
+                         str(exc_context.exception))
+
+    def test_roundtrip(self):
+        self.assertEqual(
+            self.yaml, storage_to_options(options_to_storage(self.yaml)))
+        self.assertItemsEqual(
+            self.storage,
+            options_to_storage(storage_to_options(self.storage)))
 
=== modified file 'charmworld/tests/test_search.py'
--- charmworld/tests/test_search.py	2013-07-01 14:45:02 +0000
+++ charmworld/tests/test_search.py	2013-07-05 14:15:30 +0000
@@ -173,7 +173,7 @@
             elif key in ('provides', 'requires'):
                 query = query.values()[0].values()[0]
             elif key == 'config':
-                query = query.values()[0].values()[0]['description']
+                query = query.values()[0][0]['description']
             elif key == 'store_data':
                 query = query['digest']
             elif key == 'files':
 
=== modified file 'charmworld/views/api.py'
--- charmworld/views/api.py	2013-07-02 12:28:28 +0000
+++ charmworld/views/api.py	2013-07-05 14:15:30 +0000
@@ -60,7 +60,7 @@
     message = 'This API version is no longer supported.'
     return Response(
         message,
-        headerlist = [
+        headerlist=[
             ('Access-Control-Allow-Origin', '*'),
             ('Access-Control-Allow-Headers', 'X-Requested-With'),
             ('Content-Type', 'text/plain'),
 
=== modified file 'charmworld/views/charms.py'
--- charmworld/views/charms.py	2013-07-03 21:33:20 +0000
+++ charmworld/views/charms.py	2013-07-05 14:15:30 +0000
@@ -165,7 +165,10 @@
 
 
 def _json_charm(charm):
+    charm_config = charm.config
     charm = dict(charm._representation)
+    # Reformat config into old internal representation.
+    charm['config'] = charm_config
     charm_keys = set(charm.keys())
     for k in SANITIZE.intersection(charm_keys):
         del charm[k]
 
=== modified file 'charmworld/views/tests/test_charms.py'
--- charmworld/views/tests/test_charms.py	2013-07-03 21:33:20 +0000
+++ charmworld/views/tests/test_charms.py	2013-07-05 14:15:30 +0000
@@ -1,6 +1,7 @@
 # Copyright 2012, 2013 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
+from datetime import date
 from logging import getLogger
 
 from charmworld.jobs.ingest import (
@@ -16,7 +17,6 @@
 from charmworld.testing import ViewTestBase
 from charmworld.testing import WebTestBase
 from charmworld.utils import quote_key
-from charmworld.views import SANITIZE
 from charmworld.views.api import API2
 from charmworld.views.charms import (
     _charm_view,
@@ -35,7 +35,65 @@
     quality_edit,
     series_collection,
 )
-import json
+
+
+DISTRO_CHARM_DATA = {
+    'name': 'foo',
+    'series': 'precise',
+    'owner': 'bar',
+    'maintainer': 'John Doe <[e-mail]>',
+    'summary': 'Remote terminal classroom over https',
+    'description': 'foobar',
+    'store_url': 'cs:precise/foo-1',
+    'bzr_branch': '',
+    'promulgated': True,
+    'branch_deleted': False,
+    'categories': [],
+    'is_featured': False,
+    'subordinate': False,
+    'requires': {'database': {'interface': 'mongodb'}},
+    'provides': {'website': {'interface': 'https'}},
+    'config': {
+        'options': {
+            'script-interval': {
+                'default': 5,
+                'type': 'int',
+                'description': 'The interval between script runs.'
+            }
+        }
+    },
+    'ensemble': 'formula',
+    'downloads': 0,
+    'downloads_in_past_30_days': 0,
+    'files': {
+        'readme': {
+            'subdir': '',
+            'filename': 'README',
+        },
+        'install': {
+            'subdir': 'hooks',
+            'filename': 'install',
+        },
+    },
+    'tests': {},
+    'test_results': {},
+    'proof': {
+        'w': [
+            ' no README file',
+            ' (install:31) - hook accesses EC2 metadata service'
+            ' directly'
+        ]
+    },
+    'date_created': '2012-01-01',
+    'error': '',
+    'qa': None,
+    'last_change': {
+        'committer': u'John Doe <[e-mail]>',
+        'message': u'maintainer',
+        'revno': 12,
+        'created': 1343082725.06
+    }
+}
 
 
 class TestCharms(ViewTestBase):
@@ -368,17 +426,15 @@
         self.enable_routes()
         ignore, charm = factory.makeCharm(
             self.db, name='foo', owner='bar', series='precise',
-            promulgated=True)
+            description='foobar', promulgated=True,
+            date_created=date(2012, 1, 1))
         request = self.getRequest()
         request.matchdict = {
             'charm': 'foo',
             'series': 'precise',
         }
-        charm_data = dict(Charm(charm)._representation)
-        for key in SANITIZE.intersection(charm_data.keys()):
-            del charm_data[key]
         response = distro_charm_json(request)
-        self.assertEqual(charm_data, json.loads(response.body))
+        self.assertEqual(DISTRO_CHARM_DATA, response.json_body)
 
     def test_valid_others(self):
         charm = factory.makeCharm(self.db, files={})[1]
 
=== modified file 'migrations/migrate.py'
--- migrations/migrate.py	2013-06-04 15:36:45 +0000
+++ migrations/migrate.py	2013-07-05 14:15:30 +0000
@@ -7,6 +7,7 @@
 
 """
 import argparse
+from collections import namedtuple
 from datetime import datetime
 import imp
 from os import listdir
@@ -15,8 +16,13 @@
 from os.path import join
 import re
 
-from charmworld.models import getconnection
-from charmworld.models import getdb
+from pymongo.errors import OperationFailure
+
+from charmworld.models import (
+    CharmSource,
+    getconnection,
+    getdb,
+)
 from charmworld.search import ElasticSearchClient
 from charmworld.utils import get_ini
 
@@ -45,6 +51,22 @@
     return s
 
 
+class MigrationBeforeExodus(Exception):
+    pass
+
+
+class MissingExodusCollection(Exception):
+    pass
+
+
+class MissingExodusIndex(Exception):
+    pass
+
+
+class MultipleExodus(Exception):
+    pass
+
+
 class DataStore(object):
     """Communicate with the data store to determine version status."""
 
@@ -68,6 +90,19 @@
         else:
             return None
 
+    def get_current_version(self, init):
+        current_version = self.current_version
+
+        if current_version is not None:
+            return current_version
+
+        if init:
+            # We want to auto init the database anyway.
+            self.version_datastore()
+            return 0
+        else:
+            raise Exception('Data store is not versioned')
+
     def update_version(self, version):
         """Update the version number in the data store."""
         self.db.migration_version.save({
@@ -93,6 +128,9 @@
         })
 
 
+VersionInfo = namedtuple('VersionInfo', ['num', 'filename', 'is_exodus'])
+
+
 class Versions(object):
 
     def __init__(self, path):
@@ -103,7 +141,7 @@
         self.versions_dir = path
         # Create temporary list of files, allowing skipped version numbers.
         files = listdir(path)
-        versions = {}
+        self.versions = {}
 
         for name in files:
             if not name.endswith('.py'):
@@ -113,80 +151,149 @@
             match = FILENAME_WITH_VERSION.match(name)
             if match:
                 num = int(match.group(1))
-                versions[num] = name
+                self.versions[num] = name
             else:
                 pass  # Must be a helper file or something, let's ignore it.
 
-        self.versions = {}
-        version_numbers = versions.keys()
-        version_numbers.sort()
-
-        self.version_indexes = version_numbers
-        for idx in version_numbers:
-            self.versions[idx] = versions[idx]
-
-    def create_new_version_file(self, description):
+    def iter_versions(self, current):
+        return (i for i in sorted(self.versions.items()) if i[0] > current)
+
+    def version_path(self, description, version):
+        filename = "{0}_{1}.py".format(
+            str(version).zfill(3),
+            str_to_filename(description)
+        )
+        path = join(self.versions_dir, filename)
+        return path, filename
+
+    def create_new_version_file(self, description, content=None):
         """Create Python files for new version"""
         version = self.latest + 1
 
         if not description:
             raise ValueError('Please provide a short migration description.')
-
-        filename = "{0}_{1}.py".format(
-            str(version).zfill(3),
-            str_to_filename(description)
-        )
-
-        filepath = join(self.versions_dir, filename)
-        with open(filepath, 'w') as new_migration:
-            new_migration.write(
-                SCRIPT_TEMPLATE.format(description=description))
+        if content is None:
+            content = SCRIPT_TEMPLATE.format(description=description)
+        path, filename = self.version_path(description, version)
+        with open(path, 'w') as new_migration:
+            new_migration.write(content)
 
         # Update our current store of version data.
         self.versions[version] = filename
-        self.version_indexes.append(version)
 
         return filename
 
     @property
     def latest(self):
         """:returns: Latest version in Collection"""
-        return self.version_indexes[-1] if len(self.version_indexes) > 0 else 0
-
-    def run_migration(self, db, index_client, module_name):
-        module = imp.load_source(
+        return max(self.versions.keys()) if len(self.versions) > 0 else 0
+
+    def get_exodus(self, filename):
+        module = self.import_version(filename)
+        return getattr(module, 'exodus_update', None)
+
+    def list_pending(self, current):
+        return [VersionInfo(n, f, self.get_exodus(f) is not None) for n, f in
+                self.iter_versions(current)]
+
+    def _check_exodus(self, version_info):
+        exodus_seen = False
+        migration_seen = False
+        for num, filename, is_exodus in version_info:
+            if is_exodus:
+                if migration_seen:
+                    raise MigrationBeforeExodus(
+                        'There are migrations scheduled before an exodus.')
+                if exodus_seen:
+                    raise MultipleExodus(
+                        'There are multiple exoduses scheduled.')
+                exodus_seen = True
+            else:
+                migration_seen = True
+
+    def import_version(self, module_name):
+        return imp.load_source(
             module_name.strip('.py'),
             join(self.versions_dir, module_name))
+
+    def run_migration(self, db, index_client, module_name):
+        module = self.import_version(module_name)
         getattr(module, 'upgrade')(db, index_client)
 
+    def get_pending_source(self, source, number):
+        db = source.collection.database
+        pending_suffix = '_pending_%03d' % number
+        pending_name = source.index_client.index_name + pending_suffix
+        pending_index = source.index_client.get_aliasable_client(
+            pending_name)
+        pending_collection = db[source.collection.name + pending_suffix]
+        return CharmSource(db, pending_index, pending_collection)
+
+    def prepare_exodus(self, source, init=False):
+        db = source.collection.database
+        datastore = DataStore(db)
+        current_version = datastore.get_current_version(init)
+        pending_migrations = self.list_pending(current_version)
+        self._check_exodus(pending_migrations)
+        if len(pending_migrations) == 0 or not pending_migrations[0].is_exodus:
+            return None
+        number, filename, is_exodus = pending_migrations[0]
+        pending = self.get_pending_source(source, number)
+        temp_collection = db[pending.collection.name + 'temp']
+        with source.index_client.replacement() as temp_index:
+            db.create_collection(temp_collection.name)
+            temp = CharmSource(db, temp_index, temp_collection)
+            self.get_exodus(filename)(source, temp)
+            aliased = pending.index_client.get_aliased()
+            temp_collection.rename(pending.collection.name, dropTarget=True)
+            pending.index_client.update_aliased(temp_index.index_name, aliased)
+        return pending
+
+    def finalize_exodus(self, source, version_number):
+        pending = self.get_pending_source(source, version_number)
+        aliased = source.index_client.get_aliased()
+        pending_aliased = pending.index_client.get_aliased()
+        if len(pending_aliased) == 0:
+            raise MissingExodusIndex(
+                'Exodus index "%s" does not exist.' %
+                pending.index_client.index_name)
+        if len(pending_aliased) != 1:
+            raise AssertionError('More than index aliased to pending.')
+        try:
+            pending.collection.rename(source.collection.name,
+                                      dropTarget=True)
+        except OperationFailure as e:
+            if e.message.endswith('source namespace does not exist'):
+                raise MissingExodusCollection(
+                    'Exodus collection "%s" does not exist.' %
+                    pending.collection.name)
+            raise
+        source.index_client.update_aliased(
+            pending_aliased[0], aliased)
+        for alias_name in aliased:
+            source.index_client.get_related_client(alias_name).delete_index()
+
     def upgrade(self, datastore, index_client, init):
         """Run `upgrade` methods for required version files.
 
         :param datastore: An instance of DataStore
 
         """
-        current_version = datastore.current_version
-
-        if current_version is None:
-            if init:
-                # We want to auto init the database anyway.
-                datastore.version_datastore()
-                current_version = 0
+        current_version = datastore.get_current_version(init)
+        pending_versions = self.list_pending(current_version)
+        self._check_exodus(pending_versions)
+
+        if len(pending_versions) == 0:
+            return None
+
+        for num, module_name, is_exodus in pending_versions:
+            if is_exodus:
+                source = CharmSource(datastore.db, index_client)
+                self.finalize_exodus(source, pending_versions[0][0])
             else:
-                raise Exception('Data store is not versioned')
-
-        if current_version >= self.latest:
-            # Nothing to do here. All migrations processed already.
-            return None
-
-        while current_version < self.latest:
-            # Let's get processing.
-            next_version = current_version + 1
-            module_name = self.versions[next_version]
-            self.run_migration(datastore.db, index_client, module_name)
-            current_version = next_version
-            datastore.update_version(current_version)
-        return current_version
+                self.run_migration(datastore.db, index_client, module_name)
+            datastore.update_version(num)
+        return num
 
 
 def parse_args():
@@ -214,6 +321,13 @@
         default=False,
         help='Auto init the database if not already init.')
 
+    parser_prepare_upgrade = subparsers.add_parser('prepare-upgrade')
+    parser_prepare_upgrade.set_defaults(func=Commands.prepare_upgrade)
+    parser_prepare_upgrade.add_argument(
+        '-i', '--init', action='store_true',
+        default=False,
+        help='Auto init the database if not already init.')
+
     args = parser.parse_args()
     return args
 
@@ -253,6 +367,12 @@
         print "Created new migration: {0}".format(filename)
 
     @classmethod
+    def prepare_upgrade(cls, ini, args):
+        source = CharmSource.from_settings(ini)
+        migrations = Versions(ini['migrations'])
+        migrations.prepare_exodus(source, args.init)
+
+    @classmethod
     def upgrade(cls, ini, args):
         """Upgrade the data store to the latest available migration."""
         ds = cls.get_datastore(ini)
 
=== modified file 'migrations/test_migrate.py'
--- migrations/test_migrate.py	2013-06-04 15:36:45 +0000
+++ migrations/test_migrate.py	2013-07-05 14:15:30 +0000
@@ -5,16 +5,29 @@
 from os.path import join
 from os.path import exists
 import pymongo
-from tempfile import mkdtemp
+from textwrap import dedent
 
-from charmworld.models import getconnection
-from charmworld.models import getdb
-from charmworld.testing import TestCase
+from charmworld.models import (
+    CharmSource,
+    getconnection,
+    getdb,
+)
+from charmworld.testing import (
+    MongoTestBase,
+    TestCase,
+    temp_dir
+)
 from charmworld.testing.factory import random_string
 from charmworld.utils import get_ini
 
-from migrate import DataStore
-from migrate import Versions
+from migrate import (
+    DataStore,
+    MigrationBeforeExodus,
+    MissingExodusCollection,
+    MissingExodusIndex,
+    MultipleExodus,
+    Versions,
+)
 
 INI = get_ini()
 MONGO_URL = environ.get('TEST_MONGODB')
@@ -92,7 +105,7 @@
 class TestVersions(TestCase):
     """The local versions of the files."""
     def setUp(self):
-        self.tmpdir = mkdtemp()
+        self.tmpdir = self.use_context(temp_dir())
         self.version_files = make_version_files(self.tmpdir, 4)
         self.versions = Versions(self.tmpdir)
 
@@ -102,9 +115,9 @@
 
     def test_latest_wo_migrations(self):
         """"""
-        tmpdir = mkdtemp()
-        versions = Versions(tmpdir)
-        self.assertEqual(versions.latest, 0)
+        with temp_dir() as tmpdir:
+            versions = Versions(tmpdir)
+            self.assertEqual(versions.latest, 0)
 
     def test_latest_version(self):
         """latest returns the number of the newest revision."""
@@ -128,6 +141,34 @@
         # And this new file should exist in our versions directory
         self.assertTrue(exists(join(self.tmpdir, self.versions.versions[5])))
 
+    def test_list_pending(self):
+        self.versions.create_new_version_file('foo', content=dedent("""
+            def exodus_update():
+                pass
+        """))
+        self.versions.create_new_version_file('foo', content=dedent("""
+        def update():
+            pass
+        """))
+        self.assertEqual([
+            (5, '005_foo.py', True),
+            (6, '006_foo.py', False),
+        ], self.versions.list_pending(4))
+
+    def test_check_exodus(self):
+        self.versions._check_exodus([])
+        with self.assertRaises(MigrationBeforeExodus):
+            self.versions._check_exodus([
+                (1, '001_foo.py', False), (2, '002_bar.py', True)])
+        self.versions._check_exodus([
+            (1, '001_foo.py', True), (2, '002_bar.py', False)])
+        with self.assertRaises(MultipleExodus):
+            self.versions._check_exodus([
+                (1, '001_foo.py', True), (2, '002_bar.py', True)])
+        self.versions._check_exodus([
+            (1, '001_foo.py', False), (2, '002_bar.py', False)])
+
+
 
 class TestVersionUpgrades(TestCase):
     """Functional testing of the actual upgrade process."""
@@ -136,8 +177,9 @@
             'mongo.url': MONGO_URL,
         })
         self.db = getdb(self.connection, 'juju_test')
+        self.use_index_client(aliased=True)
         self.ds = DataStore(self.db)
-        tmpdir = mkdtemp()
+        tmpdir = self.use_context(temp_dir())
         make_version_files(tmpdir, 4, runnable=True)
         self.version = Versions(tmpdir)
 
@@ -155,9 +197,8 @@
 
     def test_upgrade_wo_migrations(self):
         """Upgrade runs silently if no migrations are created yet."""
-        self.use_index_client()
         self.ds.version_datastore()
-        tmpdir = mkdtemp()
+        tmpdir = self.use_context(temp_dir())
         versions = Versions(tmpdir)
         self.assertEqual(versions.upgrade(self.ds, self.index_client, False),
                          None)
@@ -165,7 +206,6 @@
     def test_upgrade_succeeds(self):
         """upgrade will run the methods and we get a version of 4."""
         self.ds.version_datastore()
-        self.use_index_client()
         # We should get 4 versions done. The upgrade returns 4
         last_version = self.version.upgrade(self.ds, self.index_client, False)
         self.assertEqual(4, last_version)
@@ -175,10 +215,180 @@
 
     def test_upgrade_succeeds_from_unversioned(self):
         """Forcing init no an unversioned db will upgrade successfully."""
-        self.use_index_client()
         last_version = self.version.upgrade(self.ds, self.index_client, True)
         self.assertEqual(4, last_version)
 
         # and we can query the db for the revision of 4
         self.assertEqual(4, self.ds.current_version)
 
+    def create_exodus_environment(self, index_client=None):
+        if index_client is None:
+            index_client = self.index_client
+        versions = Versions(self.use_context(temp_dir()))
+        versions.create_new_version_file(
+            random_string(20), content=dedent("""
+                def exodus_update(source, target):
+                    pass
+        """))
+        source = CharmSource(self.db, index_client)
+        return versions, source
+
+    def test_upgrade_errors_if_no_exodus_db(self):
+        versions, source = self.create_exodus_environment()
+        aliased, = source.index_client.get_aliased()
+        pending = versions.get_pending_source(source, 1)
+        pending_alias = self.create_aliased(pending.index_client)
+        self.addCleanup(pending_alias.delete_index)
+        pending.index_client.wait_for_startup()
+        with self.assertRaises(MissingExodusCollection):
+            versions.upgrade(self.ds, source.index_client, True)
+        self.assertEqual([aliased], source.index_client.get_aliased())
+
+    def test_upgrade_errors_if_no_exodus_index(self):
+        versions, source = self.create_exodus_environment()
+        pending = versions.get_pending_source(source, 1)
+        pending.create_collection()
+        self.addCleanup(pending.collection.drop)
+        with self.assertRaises(MissingExodusIndex):
+            versions.upgrade(self.ds, self.index_client, True)
+        self.assertIn(pending.collection.name,
+                      pending.collection.database.collection_names())
+
+    def create_aliased(self, index_client):
+        alias_client = index_client.get_aliasable_client()
+        alias_client.create_index()
+        index_client.update_aliased(alias_client.index_name, [])
+        return alias_client
+
+    def create_exodus_success_environment(self):
+        versions, source = self.create_exodus_environment()
+        pending = versions.get_pending_source(source, 1)
+        pending.create_collection()
+        pending_alias = self.create_aliased(pending.index_client)
+        return source, pending, versions
+
+    def test_finalize_exodus_success(self):
+        source, pending, versions = self.create_exodus_success_environment()
+        aliased = source.index_client.get_aliased()
+        pending.save({'_id': 'a', 'message': 'hello'})
+        versions.finalize_exodus(source, 1)
+        for charm in source._get_all('a'):
+            self.assertEqual({'_id': 'a', 'message': 'hello'}, charm)
+        self.assertNotIn(pending.collection.name,
+                         pending.collection.database.collection_names())
+        for aliased_name in aliased:
+            aliased_client = source.index_client.get_related_client(
+                aliased_name)
+            self.assertFalse(aliased_client.exists())
+
+    def test_upgrade_succeeds_with_exodus(self):
+        source, pending, versions = self.create_exodus_success_environment()
+        pending.save({'_id': 'a', 'message': 'hello'})
+        versions.upgrade(self.ds, source.index_client, True)
+
+    def test_upgrade_errors_on_non_initial_exodus(self):
+        versions = Versions(self.use_context(temp_dir()))
+        versions.create_new_version_file(random_string(20))
+        versions.create_new_version_file(
+            random_string(20), content=dedent("""
+                def exodus_update(source, target):
+                    pass
+        """))
+        with self.assertRaises(MigrationBeforeExodus):
+            versions.upgrade(self.ds, self.index_client, True)
+
+
+class TestExodus(MongoTestBase):
+
+    def setUp(self):
+        super(TestExodus, self).setUp()
+        self.use_index_client(aliased=True)
+        self.source = CharmSource.from_request(self)
+        self.versions = Versions(self.use_context(temp_dir()))
+
+    def test_prepare_exodus_checks_validity(self):
+        self.versions.create_new_version_file(
+            random_string(20), content=dedent("""
+                def update(source, target):
+                    pass
+        """))
+        self.versions.create_new_version_file(
+            random_string(20), content=dedent("""
+                def exodus_update(source, target):
+                    pass
+        """))
+        with self.assertRaises(MigrationBeforeExodus):
+            self.versions.prepare_exodus(self.source, init=True)
+
+    def test_prepare_exodus_initializes(self):
+        with self.assertRaises(Exception) as exc_context:
+            self.versions.prepare_exodus(self.source, init=False)
+        self.assertEqual('Data store is not versioned',
+                         str(exc_context.exception))
+        self.versions.prepare_exodus(self.source, init=True)
+        ds = DataStore(self.source.collection.database)
+        self.assertEqual(0, ds.current_version)
+
+    def test_prepare_exodus_acquires_pending(self):
+        pending = self.versions.prepare_exodus(self.source, init=True)
+        self.assertIs(None, pending)
+        self.versions.create_new_version_file('foo', content=dedent("""
+                def exodus_update(source, target):
+                    pass
+        """))
+        pending = self.versions.prepare_exodus(self.source)
+        self.addCleanup(pending.index_client.delete_index)
+        self.assertIsInstance(pending, CharmSource)
+
+    def test_prepare_exodus_runs_exodus_update(self):
+        self.versions.create_new_version_file('foo', content=dedent("""
+            def exodus_update(source, target):
+                target.save({'_id': 'a', 'message': 'hello'})
+        """))
+        pending = self.versions.prepare_exodus(self.source, init=True)
+        self.addCleanup(pending.index_client.delete_index)
+        for charm in pending._get_all('a'):
+            self.assertEqual({'_id': 'a', 'message': 'hello'}, charm)
+
+    def test_exodus_update_runs_against_temp_but_generates_pending(self):
+        self.versions.create_new_version_file('foo', content=dedent("""
+            def exodus_update(source, target):
+                if target.index_client.index_name == 'temp_index_pending_001':
+                    raise AssertionError('Used pending name!')
+                if target.collection.name == 'charms_pending_001':
+                    raise AssertionError('Used pending name!')
+        """))
+        pending = self.versions.prepare_exodus(self.source, init=True)
+        self.addCleanup(pending.index_client.delete_index)
+        self.assertEqual('temp_index_pending_001',
+            pending.index_client.index_name)
+        self.assertEqual('charms_pending_001', pending.collection.name)
+
+    def test_no_leftovers(self):
+        db = self.source.collection.database
+        index_client = self.source.index_client._client
+        index_keys = index_client.status()['indices'].keys()
+        index_count = len(index_client.status()['indices'])
+        DataStore(db).version_datastore()
+        self.source.create_collection()
+        collection_names = db.collection_names()
+        self.versions.create_new_version_file('foo', content=dedent("""
+            def exodus_update(source, target):
+                pass
+        """))
+        self.versions.prepare_exodus(self.source, init=True)
+        self.versions.finalize_exodus(self.source, 1)
+        self.assertEqual(collection_names, db.collection_names())
+        self.assertEqual(index_count, len(index_client.status()['indices']))
+
+    def test_suffix_of_source(self):
+        self.versions.create_new_version_file('foo', content=dedent("""
+            import re
+
+            def exodus_update(source, target):
+                target_name = target.index_client.index_name
+                match = re.match('(.*)-([0-9]*)', target_name)
+                if match.group(1) != source.index_client.index_name:
+                    raise AssertionError('Poor name: ' + target_name)
+        """))
+        self.versions.prepare_exodus(self.source, init=True)
 
=== removed file 'migrations/versions/005_no_op.py'
--- migrations/versions/005_no_op.py	2013-06-04 18:22:19 +0000
+++ migrations/versions/005_no_op.py	1970-01-01 00:00:00 +0000
@@ -1,5 +0,0 @@
-# Copyright 2013 Canonical Ltd.  This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-def upgrade(db, index_client):
-    """005 failed to remove icons from charms, so now disabled."""
 
=== removed file 'migrations/versions/006_no_op.py'
--- migrations/versions/006_no_op.py	2013-06-05 18:30:47 +0000
+++ migrations/versions/006_no_op.py	1970-01-01 00:00:00 +0000
@@ -1,5 +0,0 @@
-# Copyright 2013 Canonical Ltd.  This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-def upgrade(db, index_client):
-    """006 lacked a corresponding ingest update, so now disabled."""
 
=== removed file 'migrations/versions/007_no_op.py'
--- migrations/versions/007_no_op.py	2013-06-11 19:13:07 +0000
+++ migrations/versions/007_no_op.py	1970-01-01 00:00:00 +0000
@@ -1,6 +0,0 @@
-# Copyright 2013 Canonical Ltd.  This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-
-def upgrade(db, index_client):
-    """007 was renamed to 008 because of a deployment issue in production"""
 
=== added file 'migrations/versions/010_remap_options.py'
--- migrations/versions/010_remap_options.py	1970-01-01 00:00:00 +0000
+++ migrations/versions/010_remap_options.py	2013-07-05 14:15:30 +0000
@@ -0,0 +1,17 @@
+# Copyright 2013 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from charmworld.models import (
+    options_to_storage,
+)
+
+
+def exodus_update(source, target):
+    for charm in source.collection.find():
+        config = charm.get('config')
+        if config is not None and 'options' in config:
+            config['options'] = options_to_storage(config['options'])
+        try:
+            target.save(charm)
+        except Exception as e:
+            print e
 
=== modified file 'migrations/versions/tests/test_migrations.py'
--- migrations/versions/tests/test_migrations.py	2013-06-11 19:18:47 +0000
+++ migrations/versions/tests/test_migrations.py	2013-07-05 14:15:30 +0000
@@ -1,34 +1,95 @@
 # Copyright 2012, 2013 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
+from pyelasticsearch.exceptions import (
+    ElasticHttpNotFoundError,
+)
 from charmworld.models import CharmSource
 from charmworld.testing import MongoTestBase
 from migrations.migrate import Versions
 
 
-def run_migration(db, index_client, module_name):
-    versions = Versions('migrations/versions/')
-    versions.run_migration(db, index_client, module_name)
-
-
-class TestMigration004(MongoTestBase):
+class MigrationTestBase(MongoTestBase):
+
+    def setUp(self):
+        super(MigrationTestBase, self).setUp()
+        self.versions = Versions('migrations/versions/')
+
+
+class TestMigration004(MigrationTestBase):
 
     def test_migration(self):
         self.use_index_client()
         self.db.create_collection('charm-errors')
-        run_migration(self.db, self.index_client, '004_remove_charm_errors.py')
+        self.versions.run_migration(self.db, self.index_client,
+                                    '004_remove_charm_errors.py')
         self.assertNotIn('charm-errors', self.db.collection_names())
 
 
-class TestMigration008(MongoTestBase):
+class TestMigration008(MigrationTestBase):
 
     def test_migration(self):
         self.use_index_client()
         source = CharmSource.from_request(self)
         source.save({'_id': 'a', 'icon': 'asdf', 'asdf': 'asdf'})
         source.save({'_id': 'b', 'icon': 'asdf', 'asdf': 'asdf'})
-        run_migration(self.db, self.index_client, '008_delete_icon_field.py')
-        for charm in source._get_all('a'):
-            self.assertNotIn('icon', charm)
-        for charm in source._get_all('b'):
-            self.assertNotIn('icon', charm)
+        self.versions.run_migration(self.db, self.index_client,
+                                    '008_delete_icon_field.py')
+        for charm in source._get_all('a'):
+            self.assertNotIn('icon', charm)
+        for charm in source._get_all('b'):
+            self.assertNotIn('icon', charm)
+
+
+class TestMigration010(MigrationTestBase):
+
+    def test_migration(self):
+        self.use_index_client()
+        source = CharmSource.from_request(self)
+        source.save({
+            '_id': 'a',
+            'config': None,
+        })
+        source.save({
+            '_id': 'b',
+            'config': {
+                'options': {},
+            },
+        })
+        source.save({
+            '_id': 'c',
+            'config': {
+                'options': {'foo': {'default': 'bar'}},
+            },
+        })
+        source.save({'_id': 'd'})
+        # This is not acceptable to Elasticsearch, because config is a list.
+        self.db.charms.save({
+            '_id': 'e',
+            'config': ['a'],
+        })
+        self.versions.get_exodus('010_remap_options.py')(source, source)
+        for charm in source._get_all('a'):
+            self.assertEqual({
+            '_id': 'a',
+            'config': None,
+        }, charm)
+        for charm in source._get_all('b'):
+            self.assertEqual({
+                '_id': 'b',
+                'config': {'options': []},
+        }, charm)
+        for charm in source._get_all('c'):
+            self.assertEqual({
+                '_id': 'c',
+                'config': {'options': [{
+                    'name': 'foo',
+                    'default': 'bar',
+                }]}
+            }, charm)
+        for charm in source._get_all('d'):
+            self.assertEqual({'_id': 'd'}, charm)
+        with self.assertRaises(ElasticHttpNotFoundError):
+            self.index_client.get('e')
+        self.assertEqual({'_id': 'e', 'config': ['a']},
+                         self.db.charms.find_one({'_id': 'e'}))