Merge lp:~abentley/charmworld/bulk-insert into lp:~juju-jitsu/charmworld/trunk

[Aaron Bentley]

Use bulk-insert to optimize reindexing.

Reindexing charms takes multiple seconds to complete. Ideally, this would be a faster operation, since may be needed when the code changes. By switching from insert to bulk-insert, the operation completes in sub-second time for 1000 charms.

For some reason, Elasticsearch treats a zero-length array as an error instead of a no-op, so ElasticSearchClient.index_charms special-cases it.

[Abel Deuring (adeuring)]

looks good

review: Approve (code)

=== modified file 'charmworld/search.py'
--- charmworld/search.py	2013-05-17 14:02:17 +0000
+++ charmworld/search.py	2013-05-22 13:15:34 +0000
@@ -150,6 +150,12 @@
         self._client.index(self.index_name, 'charm', charm, charm_id,
                            refresh=True)
 
+    def index_charms(self, charms):
+        if len(charms) == 0:
+            return
+        self._client.bulk_index(self.index_name, 'charm', charms, '_id',
+                                refresh=True)
+
     @staticmethod
     def _get_text_query(text):
         boosted_fields = [field + ('' if boost is None else '^%d' % boost)
@@ -359,8 +365,7 @@
         copy.create_index()
         try:
             copy.put_mapping()
-            for charm in self.api_search(valid_charms_only=False):
-                copy.index_charm(charm)
+            self.index_charms(self.api_search(valid_charms_only=False))
             return copy
         except:
             copy.delete_index()

=== modified file 'charmworld/tests/test_search.py'
--- charmworld/tests/test_search.py	2013-05-17 17:05:49 +0000
+++ charmworld/tests/test_search.py	2013-05-22 13:15:34 +0000
@@ -61,6 +61,21 @@
         self.index_client.index_charm(foo_charm)
         self.assertTrue(self.exists_in_index(foo_charm['_id']))
 
+    def test_index_charms(self):
+        foo_charm = factory.get_charm_json()
+        bar_charm = factory.get_charm_json()
+        self.assertFalse(self.exists_in_index(foo_charm['_id']))
+        self.assertFalse(self.exists_in_index(bar_charm['_id']))
+        self.index_client.index_charms([foo_charm, bar_charm])
+        self.assertTrue(self.exists_in_index(foo_charm['_id']))
+        self.assertTrue(self.exists_in_index(bar_charm['_id']))
+
+    def test_index_charms_no_charms(self):
+        # Assert that no exception is raised.
+        with self.assertRaises(AssertionError):
+            with self.assertRaises(BaseException):
+                self.index_client.index_charms([])
+
     def makeCharm(self, *args, **kwargs):
         charm = factory.get_charm_json(*args, **kwargs)
         self.index_client.index_charm(charm)