Merge lp:~ensoft-opensource/ensoft-sextant/perf into lp:ensoft-sextant

[Patrick Stevens]

Speed up program upload by batching queries differently

Upload now happens in two passes:
 - Function nodes are added.
 - Links between nodes are added.

Speedup in program upload (vim uploads three times faster). This is achieved by executing the relevant blueprint.

[lp:~ensoft-opensource/ensoft-sextant/perf]

18. By Patrick Stevens <email address hidden> on 2014-09-02

Remove old code

19. By Patrick Stevens <email address hidden> on 2014-09-03

Add a comment

[Phil Connell (pconnell)]

review: Approve

=== modified file 'src/sextant/db_api.py'
--- src/sextant/db_api.py	2014-08-27 09:07:39 +0000
+++ src/sextant/db_api.py	2014-09-03 14:12:18 +0000
@@ -67,7 +67,8 @@
         self.program_name = program_name
         self.parent_database_connection = sextant_connection
         self._functions = {}
-        self._new_tx = None
+        self._funcs_tx = None  # transaction for uploading functions
+        self._calls_tx = None  # transaction for uploading relationships
 
         if self.parent_database_connection:
             # we'll locally use db for short
@@ -80,10 +81,37 @@
                                               date=date)
             self._parent_id = parent_function.id
 
-            self._new_tx = db.transaction(using_globals=False, for_query=True)
+            self._funcs_tx = db.transaction(using_globals=False, for_query=True)
+            self._calls_tx = db.transaction(using_globals=False, for_query=True)
 
         self._connections = []
 
+    @staticmethod
+    def _get_display_name(function_name):
+        """
+        Gets the name we will display to the user for this function name.
+
+        For instance, if function_name were __libc_start_main@plt, we would
+        return ("__libc_start_main", "plt_stub"). The returned function type is
+        currently one of "plt_stub", "function_pointer" or "normal".
+
+        :param function_name: the name straight from objdump of a function
+        :return: ("display name", "function type")
+
+        """
+
+        if function_name[-4:] == "@plt":
+            display_name = function_name[:-4]
+            function_group = "plt_stub"
+        elif function_name[:20] == "_._function_pointer_":
+            display_name = function_name
+            function_group = "function_pointer"
+        else:
+            display_name = function_name
+            function_group = "normal"
+
+        return display_name, function_group
+
     def add_function(self, function_name):
         """
         Adds a function to the program, ready to be sent to the remote database.
@@ -98,21 +126,14 @@
         if self.class_contains_function(function_name):
             return True
 
-        if function_name[-4:] == "@plt":
-            display_name = function_name[:-4]
-            function_group = "plt_stub"
-        elif function_name[:20] == "_._function_pointer_":
-            display_name = function_name
-            function_group = "function_pointer"
-        else:
-            display_name = function_name
-            function_group = "normal"
+        display_name, function_group = self._get_display_name(function_name)
 
         query = ('START n = node({}) '
-                 'CREATE (n)-[:subject]->(m:func {{type: "{}", name: "{}"}})')
+                 'CREATE (n)-[:subject]->(m:func {{type: "{}", name: "{}"}}) '
+                 'RETURN m.name, id(m)')
         query = query.format(self._parent_id, function_group, display_name)
 
-        self._new_tx.append(query)
+        self._funcs_tx.append(query)
 
         self._functions[function_name] = function_name
 
@@ -159,13 +180,6 @@
             self.add_function(fn_calling)
 
         if not self.class_contains_call(fn_calling, fn_called):
-            query = ('START p = node({}) '
-                     'MATCH (p)-[:subject]->(n) WHERE n.name = "{}" '
-                     'MATCH (p)-[:subject]->(m) WHERE m.name = "{}" '
-                     'CREATE (n)-[:calls]->(m)')
-            query = query.format(self._parent_id, fn_calling, fn_called)
-            self._new_tx.append(query)
-
             self._connections.append((fn_calling, fn_called))
 
         return True
@@ -175,7 +189,24 @@
         Call this when you are finished with the object.
         Changes are not synced to the remote database until this is called.
         """
-        self._new_tx.commit()
+        functions = self._funcs_tx.commit()  # send off the function names
+
+        # now functions is a list of QuerySequence objects, which each have a
+        # .elements property which produces [['name', id]]
+
+        id_funcs = dict([seq.elements[0] for seq in functions])
+        logging.info('Functions uploaded. Uploading calls...')
+
+        # so id_funcs is a dict with id_funcs['name'] == id
+        for call in self._connections:
+            query = ('MATCH n WHERE id(n) = {} '
+                     'MATCH m WHERE id(m) = {} '
+                     'CREATE (n)-[:calls]->(m)')
+            query = query.format(id_funcs[self._get_display_name(call[0])[0]],
+                                 id_funcs[self._get_display_name(call[1])[0]])
+            self._calls_tx.append(query)
+
+        self._calls_tx.commit()
 
 
 class FunctionQueryResult: