Merge lp:~ben-hutchings/ensoft-sextant/file-names into lp:ensoft-sextant

[Ben Hutchings]

Added 'file' attribute to functions in the database, modified the objdump call code and the objdump parser to read files from the disassembley.

Functions now have a 'file' attribute in the database - which stores the path of their source file. This information is not yet used anywhere else in Sextant.

[lp:~ben-hutchings/ensoft-sextant/file-names]

32. By Ben Hutchings on 2014-11-05

fixed test_parser to run cleanly

=== modified file 'src/sextant/db_api.py'
--- src/sextant/db_api.py	2014-10-23 11:15:48 +0000
+++ src/sextant/db_api.py	2014-11-05 16:09:28 +0000
@@ -159,7 +159,7 @@
         tmp_path = os.path.join(self._tmp_dir, '{}_{{}}'.format(program_name))
 
         self.func_writer = CSVWriter(tmp_path.format('funcs'), 
-                                     headers=['name', 'type'], 
+                                     headers=['name', 'type', 'file'], 
                                      max_rows=5000)
         self.call_writer = CSVWriter(tmp_path.format('calls'), 
                                      headers=['caller', 'callee'], 
@@ -171,7 +171,7 @@
                  ' WITH line, toInt(line.id) as lineid'
                  ' MATCH (n:program {{name: "{}"}})'
                  ' CREATE (n)-[:subject]->(m:func {{name: line.name,'
-                 ' id: lineid, type: line.type}})')
+                 ' id: lineid, type: line.type, file: line.file}})')
         
         self.add_call_query = (' USING PERIODIC COMMIT 250'
                  ' LOAD CSV WITH HEADERS FROM "file:{}" AS line'
@@ -203,7 +203,7 @@
         # Propagate the error if there is one.
         return False if etype is not None else True
 
-    def add_function(self, name, typ='normal'):
+    def add_function(self, name, typ='normal', source='unknown'):
         """
         Add a function.
 
@@ -219,7 +219,7 @@
                     pointer: we know only that the function exists, not its
                             name or details.
         """
-        self.func_writer.write(name, typ)
+        self.func_writer.write(name, typ, source)
 
     def add_call(self, caller, callee):
         """
@@ -290,6 +290,7 @@
 
         tx.append('CREATE CONSTRAINT ON (p:program) ASSERT p.name IS UNIQUE')
         tx.append('CREATE INDEX ON :func(name)')
+        tx.append('CREATE INDEX ON: func(file)')
 
         # Apply the transaction.
         tx.commit()

=== modified file 'src/sextant/objdump_parser.py'
--- src/sextant/objdump_parser.py	2014-10-23 11:15:48 +0000
+++ src/sextant/objdump_parser.py	2014-11-05 16:09:28 +0000
@@ -107,8 +107,8 @@
         self._known_stubs = set()
 
         # By default print information to stdout.
-        def print_func(name, typ):
-            print('func {:25}{}'.format(name, typ))
+        def print_func(name, typ, source='unknown'):
+            print('func {:25}{:15}{}'.format(name, typ, source))
 
         def print_call(caller, callee):
             print('call {:25}{:25}'.format(caller, callee))
@@ -116,7 +116,6 @@
         def print_started(parser):
             print('parse started: {}[{}]'.format(self.path, ', '.join(self.sections)))
 
-
         def print_finished(parser):
             print('parsed {} functions and {} calls'.format(self.function_count, self.call_count))
 
@@ -134,11 +133,11 @@
         self.function_ptr_count += 1
         return name
 
-    def _add_function_normal(self, name):
+    def _add_function_normal(self, name, source):
         """
         Add a function which we have full assembly code for.
         """
-        self.add_function(name, 'normal')
+        self.add_function(name, 'normal', source)
         self.function_count += 1
 
     def _add_function_ptr(self, name):
@@ -171,10 +170,20 @@
         self.started()
 
         if self._file is not None:
-            in_section = False          # if we are in one of self.sections
-            current_function = None     # track the caller for function calls
+            in_section = False          # If we are in one of self.sections.
+            current_function = None     # Track the caller for function calls.
+            to_add = False
 
             for line in self._file:
+                if to_add:
+                    file_line = line.startswith('/')
+                    source = line.split(':')[0] if file_line else 'unknown'
+                    self._add_function_normal(current_function, source)
+                    to_add = False
+
+                    if file_line:
+                        continue
+
                 if line.startswith('Disassembly'):
                     # 'Disassembly of section <name>:\n'
                     section = line.split(' ')[-1].rstrip(':\n')
@@ -194,7 +203,8 @@
                             self._add_function_stub(current_function)
                         else:
                             current_function = function_identifier
-                            self._add_function_normal(current_function)
+                            # Flag function - we look for source on the next line.
+                            to_add = True
 
                     elif 'call ' in line or 'callq ' in line:
                         # WHITESPACE to prevent picking up function names 
@@ -273,11 +283,17 @@
             The path of the file to run objdump on.
         
     """
+    print('input file: {}'.format(input_file))
     # A single section can be specified for parsing with the -j flag,
     # but it is not obviously possible to parse multiple sections like this.
-    p = subprocess.Popen(['objdump', '-d', input_file, '--no-show-raw-insn'], 
+    p = subprocess.Popen(['objdump', '-d', input_file,
+                          '--no-show-raw-insn', '--line-numbers'], 
                          stdout=subprocess.PIPE)
-    g = subprocess.Popen(['egrep', 'Disassembly|call(q)? |>:$'], stdin=p.stdout, stdout=subprocess.PIPE)
+    # Egrep filters out the section headers (Disassembly of section...),
+    # the call lines (... [l]call[q] ...), the function declarations
+    # (... <function>:$) and the file paths (^/file_path).
+    g = subprocess.Popen(['egrep', 'Disassembly|call(q)? |>:$|^/'], 
+                         stdin=p.stdout, stdout=subprocess.PIPE)
     return input_file, g.stdout
 
 

=== modified file 'src/sextant/test_parser.py'
--- src/sextant/test_parser.py	2014-10-23 11:15:48 +0000
+++ src/sextant/test_parser.py	2014-11-05 16:09:28 +0000
@@ -23,7 +23,7 @@
         calls = defaultdict(list)
         
         # set the Parser to put output in local dictionaries
-        add_function = lambda n, t: self.add_function(functions, n, t)
+        add_function = lambda n, t, s='unknown': self.add_function(functions, n, t)
         add_call = lambda a, b: self.add_call(calls, a, b)
 
         p = parser.Parser(path, sections=sections, ignore_ptrs=ignore_ptrs, 

=== modified file 'src/sextant/test_resources/parser_test'
Binary files src/sextant/test_resources/parser_test	2014-10-13 14:10:01 +0000 and src/sextant/test_resources/parser_test	2014-11-05 16:09:28 +0000 differ
=== modified file 'src/sextant/update_db.py'
--- src/sextant/update_db.py	2014-10-17 14:20:06 +0000
+++ src/sextant/update_db.py	2014-11-05 16:09:28 +0000
@@ -82,10 +82,10 @@
             print('done: {} functions and {} calls.'
                   .format(parser.function_count, parser.call_count))
 
-        parser = Parser(file_path = file_path, file_object = file_object,
+        parser = Parser(file_path=file_path, file_object = file_object,
                         sections=[],
-                        add_function = program.add_function,
-                        add_call = program.add_call,
+                        add_function=program.add_function,
+                        add_call=program.add_call,
                         started=lambda parser: start_parser(program),
                         finished=lambda parser: finish_parser(parser, program))
         parser.parse()