Merge lp:~ensoft-opensource/ensoft-sextant/trunk into lp:ensoft-sextant

Proposed by James
Status: Merged
Approved by: Martin Morrison
Approved revision: 178
Merged at revision: 2
Proposed branch: lp:~ensoft-opensource/ensoft-sextant/trunk
Merge into: lp:ensoft-sextant
Diff against target: 3108 lines (+2792/-161)
27 files modified
LICENSE.txt (+27/-0)
MANIFEST.in (+3/-0)
Parser/parser.py (+0/-161)
bin/sextant (+8/-0)
bin/sextant.bat (+3/-0)
doc/Program_upload_docs.mkd (+9/-0)
doc/Web_server.mkd (+20/-0)
doc/wiki/Reference (+45/-0)
etc/sextant.conf (+7/-0)
resources/web/index.html (+45/-0)
resources/web/interface.html (+103/-0)
resources/web/queryjavascript.js (+201/-0)
resources/web/style_sheet.css (+36/-0)
setup.cfg (+2/-0)
setup.py (+25/-0)
src/sextant/__init__.py (+21/-0)
src/sextant/__main__.py (+220/-0)
src/sextant/db_api.py (+610/-0)
src/sextant/errors.py (+20/-0)
src/sextant/export.py (+152/-0)
src/sextant/objdump_parser.py (+272/-0)
src/sextant/pyinput.py (+180/-0)
src/sextant/query.py (+113/-0)
src/sextant/tests.py (+261/-0)
src/sextant/update_db.py (+78/-0)
src/sextant/web/__init__.py (+8/-0)
src/sextant/web/server.py (+323/-0)
To merge this branch: bzr merge lp:~ensoft-opensource/ensoft-sextant/trunk
Reviewer Review Type Date Requested Status
Phil Connell Approve
Ensoft Patch Lander Pending
Review via email: mp+230964@code.launchpad.net

Commit message

setup.py no uploads config and web file to the correct location for the pypi installation.

Description of the change

setup.py no uploads config and web file to the correct location for the pypi installation.

To post a comment you must log in.
Revision history for this message
Phil Connell (pconnell) wrote :

Merging in without a review, to get trunk in sync with latest code.

Full review to be done at some point.

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== added file 'LICENSE.txt'
2--- LICENSE.txt 1970-01-01 00:00:00 +0000
3+++ LICENSE.txt 2014-08-15 12:04:08 +0000
4@@ -0,0 +1,27 @@
5+Copyright (c) 2014, Ensoft Ltd.
6+All rights reserved.
7+
8+Redistribution and use in source and binary forms, with or without
9+modification, are permitted provided that the following conditions are met:
10+
11+1. Redistributions of source code must retain the above copyright notice, this
12+ list of conditions and the following disclaimer.
13+2. Redistributions in binary form must reproduce the above copyright notice,
14+ this list of conditions and the following disclaimer in the documentation
15+ and/or other materials provided with the distribution.
16+
17+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
18+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
19+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
20+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
21+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
22+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
23+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
24+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
26+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27+
28+The views and conclusions contained in the software and documentation are those
29+of the authors and should not be interpreted as representing official policies,
30+either expressed or implied, of the FreeBSD Project.
31+
32
33=== added file 'MANIFEST.in'
34--- MANIFEST.in 1970-01-01 00:00:00 +0000
35+++ MANIFEST.in 2014-08-15 12:04:08 +0000
36@@ -0,0 +1,3 @@
37+include *.py
38+recursive-include resources *.html *.js *.css *.gif
39+
40
41=== removed directory 'Parser'
42=== removed file 'Parser/parser.py'
43--- Parser/parser.py 2014-07-03 12:28:28 +0000
44+++ Parser/parser.py 1970-01-01 00:00:00 +0000
45@@ -1,161 +0,0 @@
46-__author__ = 'patrickas'
47-
48-import re
49-import argparse
50-
51-class parsed_object():
52- """This object has a name (which is the verbatim name like '<__libc_start_main@plt>'), a position (which
53- is the virtual memory location in hex, like '08048320', extracted from the dump), and a canonical_position
54- (which is the virtual memory location in hex but stripped of leading 0s, so it should be a unique id).
55- It also has a list what_do_i_call of parsed_objects it calls using the assembly keyword 'call'.
56- It has a list original_code of its assembler code, too, in case it's useful."""
57-
58- @staticmethod
59- def get_canonical_position(position):
60- return position.lstrip('0')
61-
62- def __eq__(self, other):
63- return self.canonical_position == other.canonical_position and self.name == other.name
64-
65- def __init__(self, input_lines, assembler_section = ''):
66- """Creates a new parsed_object given the relevant definition-lines from objdump -S.
67- A sample first definition-line is '08048300 <__gmon_start__@plt>:\n' but this function
68- expects to see the entire definition eg
69-
70-080482f0 <puts@plt>:
71- 80482f0: ff 25 00 a0 04 08 jmp *0x804a000
72- 80482f6: 68 00 00 00 00 push $0x0
73- 80482fb: e9 e0 ff ff ff jmp 80482e0 <_init+0x30>
74-
75- We also might expect assembler_section, which is for instance '.init' in 'Disassembly of section .init:'
76- """
77- self.name = re.search(r'<.+>', input_lines[0]).group(0)
78- self.position = re.search(r'^[0-9a-f]+', input_lines[0]).group(0)
79- self.canonical_position = parsed_object.get_canonical_position(self.position)
80- self.assembler_section = assembler_section
81- self.original_code = input_lines[1:]
82-
83- #todo: work out what this node calls, and store it in what_do_i_call
84- lines_where_i_call = [line for line in input_lines if re.search(r'\tcall [0-9a-f]+ <.+>\n', line)]
85-
86- self.what_do_i_call = []
87- for line in lines_where_i_call:
88- called = (re.search(r'\tcall [0-9a-f]+ <.+>\n', line).group(0))[8:]
89- address, name = called.split(' ')
90- self.what_do_i_call.append((address, name.rstrip('\n')))
91-
92-
93-
94- def __str__(self):
95- return 'Memory address ' + self.position + ' with name ' + self.name + ' in section ' + str(self.assembler_section)
96-
97- def __repr__(self):
98- out_str = 'Disassembly of section ' + self.assembler_section + ':\n\n' + self.position + ' ' + self.name + ':\n'
99- return out_str + '\n'.join([' ' + line for line in self.original_code])
100-
101-class Parser:
102- # Class to manipulate the output of objdump
103-
104- def __init__(self, input_file_location):
105- """Creates a new Parser, given an input file path. That path should be an output from objdump -S."""
106- file = open(input_file_location, 'r')
107- self.source_string_list = [line for line in file]
108- file.close()
109- self.parsed_objects = []
110-
111- def create_objects(self):
112- """ Go through the source_string_list, getting object names (like <main>) along with the corresponding
113- definitions, and put them into parsed_objects """
114-
115- parsed_objects = []
116- current_object = []
117- current_section = ''
118- for line in self.source_string_list[4:]: # we bodge, since the file starts with a little bit of guff
119- if re.match(r'[0-9a-f]+ <.+>:\n', line):
120- # we are a starting line
121- current_object = [line]
122- elif re.match(r'Disassembly of section', line):
123- current_section = re.search(r'section .+:\n', line).group(0).lstrip('section ').rstrip(':\n')
124- current_object = []
125- elif line == '\n':
126- #we now need to stop parsing the current block, and store it
127- if len(current_object) > 0:
128- parsed_objects.append(parsed_object(current_object, current_section))
129- else:
130- current_object.append(line)
131-
132- # now we should be done. We assumed that blocks begin with r'[0-9a-f]+ <.+>:\n' and end with a newline.
133- #clear duplicates:
134-
135- self.parsed_objects = []
136- for obj in parsed_objects:
137- if obj not in self.parsed_objects:
138- self.parsed_objects.append(obj)
139-
140- # by this point, each object contains a self.what_do_i_call which is a list of tuples ('address', 'name').
141-
142- def object_lookup(self, object_name = '', object_address = ''):
143- """Returns the object with name object_name or address object_address (at least one must be given).
144- If objects with the given name or address
145- are not found, returns None."""
146-
147- if object_name == '' and object_address == '':
148- return None
149-
150- trial_obj = self.parsed_objects
151-
152- if object_name != '':
153- trial_obj = [obj for obj in trial_obj if obj.name == object_name]
154-
155- if object_address != '':
156- trial_obj = [obj for obj in trial_obj if obj.canonical_position == parsed_object.get_canonical_position(object_address)]
157-
158- if len(trial_obj) == 0:
159- return None
160-
161- return trial_obj
162-
163- def link_objects(self):
164- """Goes through the objects which have already been imported, making their self.what_do_i_call into a list of
165- objects, rather than a list of (address, name)"""
166- #todo: implement this
167-
168- for obj in self.parsed_objects:
169- output_list = []
170- for called_thing in obj.what_do_i_call:
171- ref = called_thing[0]
172- if self.object_lookup(object_address=ref) is None:
173- #here, we have an object which is not found in our object_lookup
174- pass
175- else:
176- #called_thing looks like ('address', 'name'); we need to pass this list one by one into object_lookup
177- for looked_up in self.object_lookup(object_address=ref):
178- output_list.append(looked_up)
179- obj.what_do_i_call = output_list
180-
181-def main():
182-
183-
184- p = Parser(r"C:\Users\patrickas\Desktop\Objdump_parser\objdump_samples\objdump_libcopp.txt") #command line input of this, or accept a .o file (to run objdump against)
185- p.create_objects()
186-
187- print('Functions found:')
188-
189- for obj in p.parsed_objects:
190- print(str(obj))
191-
192- p.link_objects()
193-
194- func_to_investigate = '<copp_sampler_et_ht_lookup>' #do this for all functions
195-
196- print('Investigation of {}:'.format(func_to_investigate))
197-
198- #we can investigate a particular object:
199- main_func = [obj for obj in p.parsed_objects if obj.name == func_to_investigate][0]
200- print('{} calls '.format(func_to_investigate),end='')
201- print([str(f) for f in main_func.what_do_i_call])
202-
203- print('Ended investigation')
204-
205-if __name__ == "__main__":
206- main()
207\ No newline at end of file
208
209=== added directory 'bin'
210=== added file 'bin/sextant'
211--- bin/sextant 1970-01-01 00:00:00 +0000
212+++ bin/sextant 2014-08-15 12:04:08 +0000
213@@ -0,0 +1,8 @@
214+#!/bin/bash
215+
216+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
217+
218+export PYTHONPATH=${PYTHONPATH}:${DIR}/../src
219+
220+python -m sextant $@
221+
222
223=== added file 'bin/sextant.bat'
224--- bin/sextant.bat 1970-01-01 00:00:00 +0000
225+++ bin/sextant.bat 2014-08-15 12:04:08 +0000
226@@ -0,0 +1,3 @@
227+set DIR=%~dp0..\src\
228+
229+cmd /C "set PYTHONPATH=%DIR% && c:\python27\python -m sextant %*"
230
231=== added directory 'doc'
232=== added file 'doc/Instructions For yED use.docx'
233Binary files doc/Instructions For yED use.docx 1970-01-01 00:00:00 +0000 and doc/Instructions For yED use.docx 2014-08-15 12:04:08 +0000 differ
234=== added file 'doc/Program_upload_docs.mkd'
235--- doc/Program_upload_docs.mkd 1970-01-01 00:00:00 +0000
236+++ doc/Program_upload_docs.mkd 2014-08-15 12:04:08 +0000
237@@ -0,0 +1,9 @@
238+# Function pointers
239+
240+We treat function pointers as functions in their own right, but we rename them to `_._function_pointer_n` where `n` is a numerical identifier (unique to each function pointer call). When ProgramConverter is used to produce graphical output for graphs with function pointers in, they are displayed as yellow nodes with name `(function pointer)`.
241+
242+Optionally, the flag `--ignore-function-pointers` can be provided to `program_upload.py` to cause the parser to pretend function pointer calls simply do not exist.
243+
244+# Query parameters
245+We take the mandatory `--input-file`, a filepath for a program file whose call graph we wish to upload to the Neo4J server. Optionally we can specify that it is an ``--object-file` to have `objdump` run against it; otherwise, we assume the input was a text file consisting of `objdump`'s output.
246+Optional (if specified in config file) `--remote-neo4j`, the location of the Neo4J server (eg. `http://localhost:7474`).
247\ No newline at end of file
248
249=== added file 'doc/Web_server.mkd'
250--- doc/Web_server.mkd 1970-01-01 00:00:00 +0000
251+++ doc/Web_server.mkd 2014-08-15 12:04:08 +0000
252@@ -0,0 +1,20 @@
253+The server is running on Twisted. It has a static directory (`/trunk/sextant/web/public_html` in the repository) hosted at `/hello_world` for the moment. It also has `/database_properties` and `/output_graph.svg`. This page documents (so far) the dynamically generated pages.
254+
255+# database_properties
256+This page is designed to be roughly equivalent to the Python script `/graph_properties.py`. It takes arguments `query` of 'functions' or 'properties' and optionally `program_name` (which is ignored unless 'query=functions' was specified). Its return content-type is `text/plain`. The page returns a list of function names as:
257+
258+ ["func1", "func2", ... , "funcn"]
259+
260+if `query=functions` was specified, where the `funci` are the function names in the database under program-name `program_name`. It returns `None` if there was no program with the given name, and `[]` if there were no functions in the existing program with the given name.
261+
262+If instead `query=programs` was specified, we return:
263+
264+ ["prog1", ... , "progn"]
265+
266+where the `progi` are the program names stored in the database. If there are no programs in the database, we return `[]`.
267+
268+# output_graph.svg
269+This page is roughly equivalent to `/query_graph.py`. It takes arguments `program_name`, `query`. If the `program_name` was not found in the database, raises a 404 Not Found error. The `query` must be `whole_program`, `functions_calling` or `functions_called_by`; if it is not found, we use `whole_program` as a default.
270+If `query` is `functions_calling` or `functions_called_by`, we require a `func1` parameter, as the name of the function of interest.
271+
272+The page returns a GraphViz-generated file of the query run against the specified function. Its content-type is `image/svg+xml`.
273\ No newline at end of file
274
275=== added directory 'doc/wiki'
276=== added file 'doc/wiki/Reference'
277--- doc/wiki/Reference 1970-01-01 00:00:00 +0000
278+++ doc/wiki/Reference 2014-08-15 12:04:08 +0000
279@@ -0,0 +1,45 @@
280+
281+= Sextant Notes =
282+== Configuration ==
283+In the config file you should specify:
284+ * The url under which Neo4j is running,
285+ * The port on which the web interface will run,
286+ * Your definition of what is a common function (number of functions calling that function).
287+An example file is provided in the etc folder.
288+If your instance of Neo4j is local, please start running the server before uploading a program or performing queries.
289+== File Management ==
290+No more than one program can have the same name in the database.
291+=== Uploading a File ===
292+set-file-name and not-object-file are optional.
293+{{{
294+sextant add_program --input-file <name of file being input> --set-file-name <name file to be stored under in database> --not-object-file <True or False (default is False; pass through Objdump)>
295+}}}
296+=== Deleting File ===
297+It is good practice to delete files from the server since having many graphs in the database can lead to negative effects on performance.
298+{{{
299+sextant delete_program --program name <program name to be deleted as stored in database>
300+}}}
301+== Queries ==
302+=== Command Line ===
303+Command line queries produce a text output, either as a list or in GraphML which can be opened in yED.
304+All queries take the form.
305+{{{
306+sextant query --program-name <program name> --query <query name> --funcs <One or two functions as the arguments of the query> --suppress-common <True or False (default is False)>
307+}}}
308+Here common sense prevails. "return-all-program-names" doesn't have a "--program-name" argument or "--funcs" or "--suppress-common". Similarly whole-graph doesn't take any "--funcs". "--suppress-common" is always optional and can only be used for a GraphML output.
309+The options for queries are:
310+ * functions-calling,
311+ * functions-called-by,
312+ * calls-between,
313+ * whole-graph,
314+ * shortest-path,
315+ * return-all-program-names,
316+ * return-all-function-names.
317+For help with queries from the command line and the arguments type:
318+{{{
319+sextant query -h
320+}}}
321+=== Web ===
322+To run the web server type
323+{{{ sextant web }}}
324+into the command line; then navigate a web browser to the port specified.
325\ No newline at end of file
326
327=== added directory 'etc'
328=== added file 'etc/sextant.conf'
329--- etc/sextant.conf 1970-01-01 00:00:00 +0000
330+++ etc/sextant.conf 2014-08-15 12:04:08 +0000
331@@ -0,0 +1,7 @@
332+[Neo4j]
333+# URL of the Neo4J server
334+remote_neo4j = http://localhost:7474
335+# port on which to serve Sextant Web
336+port = 2905
337+# number of calls incoming before we consider a function to be 'common'
338+common_function_calls = 10
339
340=== added directory 'resources'
341=== added directory 'resources/web'
342=== added file 'resources/web/index.html'
343--- resources/web/index.html 1970-01-01 00:00:00 +0000
344+++ resources/web/index.html 2014-08-15 12:04:08 +0000
345@@ -0,0 +1,45 @@
346+<!--
347+ Sextant
348+ Copyright 2014, Ensoft Ltd.
349+ Author: Patrick Stevens
350+
351+ Home screen for Sextant-->
352+
353+<!DOCTYPE html>
354+<html>
355+<head>
356+ <meta charset="utf-8"/>
357+ <link rel="stylesheet" type="text/css" href="style_sheet.css"/>
358+ <title>Sextant</title>
359+</head>
360+<body>
361+<table>
362+<tr>
363+ <td>
364+ <h1>You have reached the portal to Sextant.</h1>
365+ </td>
366+ <td>
367+ </td>
368+</tr>
369+<tr>
370+ <td>
371+ <!--Sextant image-->
372+ <img src="sextant.gif" alt = "sextant">
373+ </td>
374+ <td>
375+ <!--link to the Query page-->
376+ <form action='/interface.html' method="get">
377+ <input type="submit" value="Query Page" class="button2"
378+ name="link_query" id="form1_submit" />
379+ </form>
380+ <!--link to Database Properties page-->
381+ <form action='/database_properties' method="get">
382+ <input type="submit" value="Database Properties" class="button2"
383+ name="link_properties" id="form2_submit" />
384+ </form>
385+ </td>
386+</tr>
387+</table>
388+
389+</body>
390+</html>
391
392=== added file 'resources/web/interface.html'
393--- resources/web/interface.html 1970-01-01 00:00:00 +0000
394+++ resources/web/interface.html 2014-08-15 12:04:08 +0000
395@@ -0,0 +1,103 @@
396+<!--
397+ Sextant
398+ Copyright 2014, Ensoft Ltd.
399+ Author: James Harkin
400+
401+ Sextant web interface for Queries -->
402+
403+<!DOCTYPE html>
404+<html>
405+<head>
406+<meta charset="utf-8"/>
407+ <link rel="stylesheet" type="text/css" href="style_sheet.css"/>
408+ <title> Webinterface test </title>
409+</head>
410+
411+<body onload="get_names_for_autocomplete('programs')">
412+<!-- Create a table with a border to hold the input file query name
413+ and arguments. table remains fixed while page scrolls-->
414+<table style="background-color:#FFFFFF; width: 100%;
415+ border:1px solid black; position: fixed;">
416+<tr style="height:20px">
417+ <td style="width:250">
418+ <h2>Input file</h2>
419+ </td>
420+ <td style="width:350">
421+ <h2>Query</h2>
422+ </td>
423+ <td style="width:250">
424+ <h2>Arguments</h2>
425+ </td>
426+ <td>
427+ <form>
428+ <!--checkbox used to supress common nodes-->
429+ <label>
430+ <input type="checkbox" id="suppress_common" value="True" />
431+ Suppress common functions?
432+ </label>
433+ </form>
434+ </td>
435+</tr>
436+<tr style="height:100px">
437+ <td>
438+ <!-- Autocomplete textbox for program names; populated on upload-->
439+ <input list="program_names" id="program_name"
440+ onblur="get_names_for_autocomplete('funcs')" style="size:20">
441+ <datalist id="program_names">
442+ </datalist>
443+ </td>
444+ <td>
445+ <!-- Drop-down box to hold choice of queries-->
446+ <form name="drop_list_1">
447+ <SELECT id="query_list" onchange="display_when();">
448+ <option value="whole_program">Whole graph</option>
449+ <option value="functions_calling">
450+ All functions calling specific function</option>
451+ <option value="functions_called_by">
452+ All functions called by a specific function</option>
453+ <option value="call_paths">
454+ All function call paths between two functions</option>
455+ <option value="shortest_path">
456+ Shortest path between two functions</option>
457+ <option value="function_names">
458+ All function names</option>
459+ </SELECT>
460+ </form>
461+ <p>
462+ query selected : <input type="text" id="query"
463+ value="Whole graph" size="20">
464+ </p>
465+ </td>
466+ <td>
467+ <!-- Autocomplete text box for argument
468+ 1 Is only visible for relevant queries-->
469+ <div id="argument_1">No arguments required.</div>
470+ <input list="function_names" id="function_1" style="size:20; visibility:hidden">
471+ <!--list to populate arguments. Updates when program name is specified-->
472+ <datalist id="function_names">
473+ </datalist>
474+ <!-- Autocomplete text box for argument
475+ 2 Is only visible for relevant queries-->
476+ <div id="argument_2"></div>
477+ <input list="function_names" id="function_2" style="size:20; visibility:hidden">
478+
479+ </td>
480+ <td>
481+
482+ <!--execute button; submits query-->
483+ <br><input type="submit" value="Execute&#x00A;Query" class="button"
484+ onclick="execute_query()"><br>
485+ </td>
486+</tr>
487+</table>
488+<!-- Paragraph for text output e.g. list of funtion names-->
489+<p id="function_names_output" class="pos_functions" style="size:20; visibility:hidden"></p>
490+
491+<!-- Output for image file-->
492+<img id=output_image src="sextant.jpg" alt="Execute query to display graph."
493+ class="pos_img"
494+ style="align: bottom; font-style: italic; color: #C0C0C0; font-size: 15px;">
495+
496+<script src="queryjavascript.js"></script>
497+</body>
498+</html>
499
500=== added file 'resources/web/queryjavascript.js'
501--- resources/web/queryjavascript.js 1970-01-01 00:00:00 +0000
502+++ resources/web/queryjavascript.js 2014-08-15 12:04:08 +0000
503@@ -0,0 +1,201 @@
504+// Sextant
505+// Copyright 2014, Ensoft Ltd.
506+// Author: James Harkin, Patrick Stevens
507+//
508+//Runs query and "GET"s either program names uploaded on the
509+//server or function names from a specific program.
510+
511+
512+function get_names_for_autocomplete(info_needed){
513+ //Function queries to database to create a list
514+ //which is used to populate the auto-complete text boxes.
515+ var xmlhttp = new XMLHttpRequest();
516+ xmlhttp.onreadystatechange = function(){
517+ if (xmlhttp.status = 200){
518+ var values_list = xmlhttp.responseText;
519+ console.log(values_list)
520+ values_list = JSON.parse(values_list);
521+ if (info_needed =='programs'){
522+ //We need to populate the program names list
523+ add_options("program_names", values_list);
524+ }
525+ if (info_needed =='funcs'){
526+ //We need to populate the functions list for the arguments
527+ add_options("function_names", values_list);
528+ }
529+ }
530+ }
531+ if (info_needed == 'programs'){
532+ var string = "/database_properties?query=" + info_needed + "&program_name=";
533+ }
534+ else{
535+ var string = "/database_properties?query=" + "functions" +
536+ "&program_name=" + document.getElementById("program_name").value;
537+ if (info_needed == 'programs'){
538+ var string = "/database_properties?query=" +
539+ info_needed + "&program_name=" + prog_name;
540+ }
541+ //"GET" information from the specified url (string)
542+ xmlhttp.open("GET", string, true);
543+ xmlhttp.send();
544+ }
545+ xmlhttp.open("GET", string, true);
546+ xmlhttp.send();
547+}
548+
549+
550+function add_options(selectedlist, values_list){
551+ //Adds all the options obtained from the list of program
552+ //names or function names to an auto complete drop-down box
553+ var options = ''
554+ if (values_list.length == 1 || values_list.length ==0){
555+ options += '<option value="'+values_list+'"/>';
556+ }
557+ else{
558+ for (var i=0; i < values_list.length;++i){
559+ options += '<option value="'+values_list[i]+'"/>';
560+ }
561+ }
562+ document.getElementById(selectedlist).innerHTML = options;
563+}
564+
565+
566+function display_when(){
567+ //For each query specifies when auto-complete text boxes should be made
568+ //visible or invisible and makes them read only
569+ var query_list = document.getElementById("query_list");
570+ document.getElementById("query").value =
571+ query_list.options[query_list.selectedIndex].text;
572+ var no_functions = new Array();
573+ var prog_name = document.getElementById("program_name").value;
574+ if (query_list.options[query_list.selectedIndex].value == "functions_calling"){
575+ document.getElementById("argument_1").innerHTML = "Function being called";
576+ document.getElementById("argument_2").innerHTML = "";
577+ document.getElementById("function_1").readOnly = false;
578+ document.getElementById("function_2").readOnly = true;
579+ document.getElementById("function_1").style.visibility = "visible";
580+ document.getElementById("function_2").style.visibility = "hidden";
581+ document.getElementById("function_2").value = null;
582+ }
583+ if (query_list.options[query_list.selectedIndex].value == "functions_called_by"){
584+ document.getElementById("argument_1").innerHTML = "Function calling";
585+ document.getElementById("argument_2").innerHTML = "";
586+ document.getElementById("function_1").readOnly = false;
587+ document.getElementById("function_2").readOnly = true;
588+ document.getElementById("function_1").style.visibility = "visible";
589+ document.getElementById("function_2").style.visibility = "hidden";
590+ document.getElementById("function_2").value = null;
591+ }
592+ if (query_list.options[query_list.selectedIndex].value == "call_paths"){
593+ document.getElementById("argument_1").innerHTML = "Function calling";
594+ document.getElementById("argument_2").innerHTML = "Function being called";
595+ document.getElementById("function_1").readOnly = false;
596+ document.getElementById("function_2").readOnly = false;
597+ document.getElementById("function_1").style.visibility = "visible";
598+ document.getElementById("function_2").style.visibility = "visible";
599+ }
600+ if (query_list.options[query_list.selectedIndex].value == "shortest_path"){
601+ document.getElementById("argument_1").innerHTML = "Function calling";
602+ document.getElementById("argument_2").innerHTML = "Function being called";
603+ document.getElementById("function_1").readOnly = false;
604+ document.getElementById("function_2").readOnly = false;
605+ document.getElementById("function_1").style.visibility = "visible";
606+ document.getElementById("function_2").style.visibility = "visible";
607+ }
608+ if (query_list.options[query_list.selectedIndex].value == "whole_program") {
609+ document.getElementById("argument_1").innerHTML = "No arguments required.";
610+ document.getElementById("argument_2").innerHTML = "";
611+ document.getElementById("function_1").readOnly = true;
612+ document.getElementById("function_2").readOnly = true;
613+ document.getElementById("function_1").style.visibility = "hidden";
614+ document.getElementById("function_2").style.visibility = "hidden";
615+ document.getElementById("function_1").value = null;
616+ document.getElementById("function_2").value = null;
617+ }
618+ if (query_list.options[query_list.selectedIndex].value == "function_names"){
619+ document.getElementById("argument_1").innerHTML = "No arguments required.";
620+ document.getElementById("argument_2").innerHTML = "";
621+ document.getElementById("function_1").readOnly = true;
622+ document.getElementById("function_2").readOnly = true;
623+ document.getElementById("function_1").style.visibility = "hidden";
624+ document.getElementById("function_2").style.visibility = "hidden";
625+ document.getElementById("function_1").value = null;
626+ document.getElementById("function_2").value = null;
627+ }
628+}
629+
630+
631+
632+function execute_query(){
633+ //Returns error in alert window if query not executed properly,
634+ //otherwise performs the query and outputs it
635+ document.getElementById("output_image").src = "";
636+ document.getElementById("output_image").alt = "Please wait loading...";
637+ var query_id = document.getElementById("query_list").value;
638+ if (query_id == "function_names"){
639+ //url for page containing all function names
640+ var string = "/database_properties?program_name=" +
641+ document.getElementById("program_name").value + "&query=functions";
642+ }
643+ else{
644+ //If not function names we will want a graph as an output;
645+ //url returns svg file of graph.
646+ var string = "/output_graph.svg?program_name=" +
647+ document.getElementById("program_name").value +
648+ "&query=" + query_id + "&func1=";
649+ string = string + document.getElementById("function_1").value +
650+ "&func2=" + document.getElementById("function_2").value;
651+ string = string + "&suppress_common=" +
652+ document.getElementById('suppress_common').checked.toString();
653+
654+ }
655+ var xmlhttp = new XMLHttpRequest();
656+ xmlhttp.open("GET", string, true);
657+ xmlhttp.send();
658+ xmlhttp.onreadystatechange = function(){
659+ if (xmlhttp.readyState == 4 && xmlhttp.status == 200){
660+ //readyState == 4 means query has finished executing.
661+ //status == 200 means "GET"ing has been successful.
662+ if (query_id == "function_names"){
663+ //Text output displayed in paragraph.
664+ document.getElementById("function_names_output").innerHTML =
665+ xmlhttp.responseText;
666+ document.getElementById("function_names_output").style.visibility =
667+ "visible"
668+ //Clear current image if one exists.
669+ document.getElementById("output_image").alt = "";
670+ document.getElementById("output_image").src = "";
671+ }
672+ else{
673+ document.getElementById("function_names_output").style.visibility =
674+ "hidden"
675+ document.getElementById("output_image").src = string;
676+ }
677+ }
678+ else if (xmlhttp.readyState == 4 && xmlhttp.status == 400){
679+ //Error occurred during query; display response.
680+ document.getElementById("output_image").alt = "";
681+ window.alert(xmlhttp.responseText);
682+ }
683+ else if(xmlhttp.readyState == 4 && xmlhttp.status == 404){
684+ //Error occurred during query; display response.
685+ document.getElementById("output_image").alt = "";
686+ window.alert(xmlhttp.responseText);
687+ }
688+ else if(xmlhttp.readyState ==4 && xmlhttp.status == 204){
689+ //Query executed correctly but graph returned is empty
690+ document.getElementById("output_image").alt = "";
691+ window.alert("Graph returned was empty");
692+ }
693+ else if (xmlhttp.readyState == 4 && xmlhttp.status == 502) {
694+ //Error occurs if Neo4j isn't running
695+ document.getElementById("output_image").alt = "";
696+ window.alert("Bad Gateway received - are you sure the database server is running?");
697+ }
698+ else if(xmlhttp.readyState ==4){
699+ //query executed correctly
700+ document.getElementById("output_image").alt = "";
701+ window.alert("Error not recognised");
702+ }
703+ }
704+}
705
706=== added file 'resources/web/sextant.gif'
707Binary files resources/web/sextant.gif 1970-01-01 00:00:00 +0000 and resources/web/sextant.gif 2014-08-15 12:04:08 +0000 differ
708=== added file 'resources/web/style_sheet.css'
709--- resources/web/style_sheet.css 1970-01-01 00:00:00 +0000
710+++ resources/web/style_sheet.css 2014-08-15 12:04:08 +0000
711@@ -0,0 +1,36 @@
712+pre {
713+ color: green;
714+ background: white;
715+ font-family: monospace;
716+}
717+
718+body {
719+ font-family: Helvetica, sans-serif;
720+}
721+.button{
722+ display:block; width:100px; height:100px; border-radius:50px; font-size:15px;
723+ color:#fff; line-height:100px; text-align:center; background:#FF0000
724+}
725+
726+.pos_img{
727+ position:absolute;
728+ left:0px;
729+ top:270px;
730+ z-index:-1
731+
732+}
733+.pos_functions{
734+ position:absolute;
735+ left:0px;
736+ top:250px;
737+
738+}
739+
740+.button2{
741+ display:inline-block; width:200px; height:100px; border-radius:50px;
742+ font-size:15px; color:#fff; line-height:100px;
743+ text-align:center; background:#000000
744+}
745+
746+
747+
748
749=== added file 'setup.cfg'
750--- setup.cfg 1970-01-01 00:00:00 +0000
751+++ setup.cfg 2014-08-15 12:04:08 +0000
752@@ -0,0 +1,2 @@
753+[metadata]
754+description-file = README.md
755\ No newline at end of file
756
757=== added file 'setup.py'
758--- setup.py 1970-01-01 00:00:00 +0000
759+++ setup.py 2014-08-15 12:04:08 +0000
760@@ -0,0 +1,25 @@
761+# -----------------------------------------
762+# Sextant
763+# Copyright 2014, Ensoft Ltd.
764+# Author: James Harkin, using work from Patrick Stevens and James Harkin
765+# -----------------------------------------
766+#
767+
768+import glob
769+import os
770+from setuptools import setup
771+
772+setup(
773+ name='Sextant',
774+ version='1.0',
775+ description= 'Navigating the C',
776+ url='http://open.ensoft.co.uk/Sextant',
777+ license='Simplified BSD License',
778+ packages=['sextant', 'sextant.web', 'resources', 'etc'],
779+ package_dir={'sextant': 'src/sextant', 'resources': 'resources', 'etc': 'etc'},
780+ scripts=['bin/sextant'],
781+ install_requires=['neo4jrestclient', 'twisted'],
782+ package_data={'resources': ['web/*'], 'etc': ['*.conf']},
783+)
784+
785+
786
787=== added directory 'src'
788=== added directory 'src/sextant'
789=== added file 'src/sextant/__init__.py'
790--- src/sextant/__init__.py 1970-01-01 00:00:00 +0000
791+++ src/sextant/__init__.py 2014-08-15 12:04:08 +0000
792@@ -0,0 +1,21 @@
793+# -----------------------------------------
794+# Sextant
795+# Copyright 2014, Ensoft Ltd.
796+# Author: Patrick Stevens, James Harkin
797+# -----------------------------------------
798+"""Program call graph recording and query framework."""
799+
800+from . import errors
801+from . import pyinput
802+
803+__all__ = (
804+ "SextantConnection",
805+) + (
806+ errors.__all__ +
807+ pyinput.__all__
808+)
809+
810+from .db_api import SextantConnection
811+from .errors import *
812+from .pyinput import *
813+
814
815=== added file 'src/sextant/__main__.py'
816--- src/sextant/__main__.py 1970-01-01 00:00:00 +0000
817+++ src/sextant/__main__.py 2014-08-15 12:04:08 +0000
818@@ -0,0 +1,220 @@
819+# -----------------------------------------
820+# Sextant
821+# Copyright 2014, Ensoft Ltd.
822+# Author: James Harkin, using work from Patrick Stevens and James Harkin
823+# -----------------------------------------
824+#invokes Sextant and argparse
825+
826+from __future__ import absolute_import, print_function
827+
828+import argparse
829+try:
830+ import ConfigParser
831+except ImportError:
832+ import configparser as ConfigParser
833+import logging
834+import logging.config
835+import os
836+import sys
837+
838+from . import update_db
839+from . import query
840+from . import db_api
841+
842+
843+# @@@ Logging level should be configurable (with some structure to setting up
844+# logging).
845+logging.config.dictConfig({
846+ "version": 1,
847+ "handlers": {
848+ "console": {
849+ "class": "logging.StreamHandler",
850+ "level": logging.INFO,
851+ "stream": "ext://sys.stderr",
852+ },
853+ },
854+ "root": {
855+ "level": logging.DEBUG,
856+ "handlers": ["console"],
857+ },
858+})
859+log = logging.getLogger()
860+
861+
862+def get_config_file():
863+ # get the config file option for neo4j server location and web port number
864+ _ROOT = os.path.abspath(os.path.dirname(__file__))
865+ def get_data(path, file_name):
866+ return os.path.join(_ROOT, path, file_name)
867+
868+ home_config = os.path.expanduser(os.path.join("~", ".sextant.conf"))
869+ env_config = os.environ.get("SEXTANT_CONFIG", "")
870+ example_config = get_data('../etc', 'sextant.conf')
871+
872+ try:
873+ conffile = next(p
874+ for p in (home_config, env_config, example_config)
875+ if os.path.exists(p))
876+ except StopIteration:
877+ #No config files accessable
878+ if "SEXTANT_CONFIG" in os.environ:
879+ #SEXTANT_CONFIG environment variable is set
880+ log.error("SEXTANT_CONFIG file %r doesn't exist.", env_config)
881+ log.error("Sextant requires a configuration file.")
882+ sys.exit(1)
883+
884+ log.info("Sextant is using config file %s", conffile)
885+ return conffile
886+
887+conffile = get_config_file()
888+
889+conf = ConfigParser.ConfigParser()
890+conf.read(conffile)
891+
892+#remote_neo4j = 'http://localhost:7474'
893+#web_port = 2905
894+#common_def = 10 # definition of a 'common' node
895+try:
896+ options = conf.options('Neo4j')
897+except ConfigParser.NoSectionError:
898+ pass
899+else:
900+ try:
901+ remote_neo4j = conf.get('Neo4j', 'remote_neo4j')
902+ except ConfigParser.NoOptionError:
903+ pass
904+
905+ try:
906+ web_port = conf.get('Neo4j', 'port')
907+ except ConfigParser.NoOptionError:
908+ pass
909+
910+ try:
911+ common_def = conf.get('Neo4j', 'common_function_calls')
912+ except ConfigParser.NoOptionError:
913+ common_def = 10
914+
915+argumentparser = argparse.ArgumentParser(description="Invoke part of the SEXTANT program")
916+subparsers = argumentparser.add_subparsers(title="subcommands")
917+
918+#set what will be defined as a "common function"
919+db_api.set_common_cutoff(common_def)
920+
921+parsers = dict()
922+
923+# add each subparser in turn to the parsers dictionary
924+
925+parsers['add'] = subparsers.add_parser('add_program', help="add a program to the database")
926+parsers['add'].add_argument('--input-file', required=True, metavar="FILE_NAME",
927+ help="name of file to be put into database",
928+ type=str, nargs=1)
929+parsers['add'].add_argument('--set-file-name', metavar="FILE_NAME",
930+ help="string to store this program under", type=str,
931+ nargs=1)
932+parsers['add'].add_argument('--not-object-file',
933+ help='default False, if the input file is an object to be disassembled',
934+ action='store_true')
935+
936+parsers['delete'] = subparsers.add_parser('delete_program',
937+ help="delete a program from the database")
938+parsers['delete'].add_argument('--program-name', required=True, metavar="PROG_NAME",
939+ help="name of program as stored in the database",
940+ type=str, nargs=1)
941+
942+parsers['query'] = subparsers.add_parser('query',
943+ help="make a query of the database")
944+parsers['query'].add_argument('--program-name', metavar="PROG_NAME",
945+ help="name of program as stored in the database",
946+ type=str, nargs=1)
947+parsers['query'].add_argument('--query', required=True, metavar="QUERY",
948+ help="functions-calling, functions-called-by, "
949+ "calls-between, whole-graph, shortest-path, "
950+ "return-all-program-names or "
951+ "return-all-function-names; if the latter, "
952+ "supply argument --program-name",
953+ type=str, nargs=1)
954+parsers['query'].add_argument('--funcs', metavar='FUNCS',
955+ help='functions to pass to the query',
956+ type=str, nargs='+')
957+parsers['query'].add_argument('--suppress-common', metavar='BOOL',
958+ help='suppress commonly called functions (True or False)',
959+ type=str, nargs=1)
960+
961+parsers['web'] = subparsers.add_parser('web', help="start the web server")
962+parsers['web'].add_argument('--port', metavar='PORT', type=int,
963+ help='port on which to serve Sextant Web',
964+ default=web_port)
965+
966+for parser_key in parsers:
967+ parsers[parser_key].add_argument('--remote-neo4j', metavar="URL", nargs=1,
968+ help="URL of neo4j server", type=str,
969+ default=remote_neo4j)
970+
971+def _start_web(args):
972+ # Don't import at top level -- this makes twisted dependency semi-optional,
973+ # allowing non-web functionality to work with Python 3.
974+ from .web import server
975+ log.info("Serving site on port %s", args.port)
976+ server.serve_site(input_database_url=args.remote_neo4j, port=args.port)
977+
978+parsers['web'].set_defaults(func=_start_web)
979+
980+def add_file(namespace):
981+
982+ try:
983+ alternative_name = namespace.set_file_name[0]
984+ except TypeError:
985+ alternative_name = None
986+
987+ not_object_file = namespace.not_object_file
988+ # the default is "yes, this is an object file" if not-object-file was
989+ # unsupplied
990+
991+ update_db.upload_program(namespace.input_file[0],
992+ namespace.remote_neo4j,
993+ alternative_name, not_object_file)
994+
995+
996+def delete_file(namespace):
997+ update_db.delete_program(namespace.program_name[0],
998+ namespace.remote_neo4j)
999+
1000+parsers['add'].set_defaults(func=add_file)
1001+parsers['delete'].set_defaults(func=delete_file)
1002+
1003+
1004+def make_query(namespace):
1005+
1006+ arg1 = None
1007+ arg2 = None
1008+ try:
1009+ arg1 = namespace.funcs[0]
1010+ arg2 = namespace.funcs[1]
1011+ except TypeError:
1012+ pass
1013+
1014+ try:
1015+ program_name = namespace.program_name[0]
1016+ except TypeError:
1017+ program_name = None
1018+
1019+ try:
1020+ suppress_common = namespace.suppress_common[0]
1021+ except TypeError:
1022+ suppress_common = False
1023+
1024+ query.query(remote_neo4j=namespace.remote_neo4j,
1025+ input_query=namespace.query[0],
1026+ program_name=program_name, argument_1=arg1, argument_2=arg2,
1027+ suppress_common=suppress_common)
1028+
1029+parsers['query'].set_defaults(func=make_query)
1030+
1031+# parse the arguments
1032+
1033+parsed = argumentparser.parse_args()
1034+
1035+# call the appropriate function
1036+
1037+parsed.func(parsed)
1038+
1039
1040=== added file 'src/sextant/db_api.py'
1041--- src/sextant/db_api.py 1970-01-01 00:00:00 +0000
1042+++ src/sextant/db_api.py 2014-08-15 12:04:08 +0000
1043@@ -0,0 +1,610 @@
1044+# -----------------------------------------
1045+# Sextant
1046+# Copyright 2014, Ensoft Ltd.
1047+# Author: Patrick Stevens, using work from Patrick Stevens and James Harkin
1048+# -----------------------------------------
1049+# API to interact with a Neo4J server: upload, query and delete programs in a DB
1050+
1051+__all__ = ("Validator", "AddToDatabase", "FunctionQueryResult", "Function",
1052+ "SextantConnection")
1053+
1054+import re # for validation of function/program names
1055+import logging
1056+
1057+from neo4jrestclient.client import GraphDatabase
1058+import neo4jrestclient.client as client
1059+
1060+COMMON_CUTOFF = 10
1061+# a function is deemed 'common' if it has more than this
1062+# many connections
1063+
1064+
1065+class Validator():
1066+ """ Sanitises/checks strings, to prevent Cypher injection attacks"""
1067+
1068+ @staticmethod
1069+ def validate(input_):
1070+ """
1071+ Checks whether we can allow a string to be passed into a Cypher query.
1072+ :param input_: the string we wish to validate
1073+ :return: bool(the string is allowed)
1074+ """
1075+ regex = re.compile(r'^[A-Za-z0-9\-:\.\$_@\*\(\)%\+,]+$')
1076+ return bool(regex.match(input_))
1077+
1078+ @staticmethod
1079+ def sanitise(input_):
1080+ """
1081+ Strips harmful characters from the given string.
1082+ :param input_: string to sanitise
1083+ :return: the sanitised string
1084+ """
1085+ return re.sub(r'[^\.\-_a-zA-Z0-9]+', '', input_)
1086+
1087+
1088+class AddToDatabase():
1089+ """Updates the database, adding functions/calls to a given program"""
1090+
1091+ def __init__(self, program_name='', sextant_connection=None):
1092+ """
1093+ Object which can be used to add functions and calls to a new program
1094+ :param program_name: the name of the new program to be created
1095+ (must already be validated against Validator)
1096+ :param sextant_connection: the SextantConnection to use for connections
1097+ """
1098+ # program_name must be alphanumeric, to avoid injection attacks easily
1099+ if not Validator.validate(program_name):
1100+ return
1101+
1102+ self.program_name = program_name
1103+ self.parent_database_connection = sextant_connection
1104+ self._functions = {}
1105+ self._new_tx = None
1106+
1107+ if self.parent_database_connection:
1108+ # we'll locally use db for short
1109+ db = self.parent_database_connection._db
1110+
1111+ parent_function = db.nodes.create(name=program_name, type='program')
1112+ self._parent_id = parent_function.id
1113+
1114+ self._new_tx = db.transaction(using_globals=False, for_query=True)
1115+
1116+ self._connections = []
1117+
1118+ def add_function(self, function_name):
1119+ """
1120+ Adds a function to the program, ready to be sent to the remote database.
1121+ If the function name is already in use, this method effectively does
1122+ nothing and returns True.
1123+
1124+ :param function_name: a string which must be alphanumeric
1125+ :return: True if the request succeeded, False otherwise
1126+ """
1127+ if not Validator.validate(function_name):
1128+ return False
1129+ if self.class_contains_function(function_name):
1130+ return True
1131+
1132+ if function_name[-4:] == "@plt":
1133+ display_name = function_name[:-4]
1134+ function_group = "plt_stub"
1135+ elif function_name[:20] == "_._function_pointer_":
1136+ display_name = function_name
1137+ function_group = "function_pointer"
1138+ else:
1139+ display_name = function_name
1140+ function_group = "normal"
1141+
1142+ query = ('START n = node({}) '
1143+ 'CREATE (n)-[:subject]->(m:func {{type: "{}", name: "{}"}})')
1144+ query = query.format(self._parent_id, function_group, display_name)
1145+
1146+ self._new_tx.append(query)
1147+
1148+ self._functions[function_name] = function_name
1149+
1150+ return True
1151+
1152+ def class_contains_function(self, function_to_find):
1153+ """
1154+ Checks whether we contain a function with a given name.
1155+ :param function_to_find: string name of the function we wish to look up
1156+ :return: bool(the function exists in this AddToDatabase)
1157+ """
1158+ return function_to_find in self._functions
1159+
1160+ def class_contains_call(self, function_calling, function_called):
1161+ """
1162+ Checks whether we contain a call between the two named functions.
1163+ :param function_calling: string name of the calling-function
1164+ :param function_called: string name of the called function
1165+ :return: bool(function_calling calls function_called in us)
1166+ """
1167+ return (function_calling, function_called) in self._connections
1168+
1169+ def add_function_call(self, fn_calling, fn_called):
1170+ """
1171+ Adds a function call to the program, ready to be sent to the database.
1172+ Effectively does nothing if there is already a function call between
1173+ these two functions.
1174+ Function names must be alphanumeric for easy security purposes;
1175+ returns False if they fail validation.
1176+ :param fn_calling: the name of the calling-function as a string.
1177+ It should already exist in the AddToDatabase; if it does not,
1178+ this method will create a stub for it.
1179+ :param fn_called: name of the function called by fn_calling.
1180+ If it does not exist, we create a stub representation for it.
1181+ :return: True if successful, False otherwise
1182+ """
1183+ if not all((Validator.validate(fn_calling),
1184+ Validator.validate(fn_called))):
1185+ return False
1186+
1187+ if not self.class_contains_function(fn_called):
1188+ self.add_function(fn_called)
1189+ if not self.class_contains_function(fn_calling):
1190+ self.add_function(fn_calling)
1191+
1192+ if not self.class_contains_call(fn_calling, fn_called):
1193+ query = ('START p = node({}) '
1194+ 'MATCH (p)-[:subject]->(n) WHERE n.name = "{}" '
1195+ 'MATCH (p)-[:subject]->(m) WHERE m.name = "{}" '
1196+ 'CREATE (n)-[:calls]->(m)')
1197+ query = query.format(self._parent_id, fn_calling, fn_called)
1198+ self._new_tx.append(query)
1199+
1200+ self._connections.append((fn_calling, fn_called))
1201+
1202+ return True
1203+
1204+ def commit(self):
1205+ """
1206+ Call this when you are finished with the object.
1207+ Changes are not synced to the remote database until this is called.
1208+ """
1209+ self._new_tx.commit()
1210+
1211+
1212+class FunctionQueryResult:
1213+ """A graph of function calls arising as the result of a Neo4J query."""
1214+
1215+ def __init__(self, parent_db, program_name='', rest_output=None):
1216+ self.program_name = program_name
1217+ self._parent_db_connection = parent_db
1218+ self.functions = self._rest_node_output_to_graph(rest_output)
1219+ self._update_common_functions()
1220+
1221+ def __eq__(self, other):
1222+ # we make a dictionary so that we can perform easy comparison
1223+ selfdict = {func.name: func for func in self.functions}
1224+ otherdict = {func.name: func for func in other.functions}
1225+
1226+ return self.program_name == other.program_name and selfdict == otherdict
1227+
1228+ def _update_common_functions(self):
1229+ """
1230+ Loop over all functions: increment the called-by count of their callees.
1231+ """
1232+ for func in self.functions:
1233+ for called in func.functions_i_call:
1234+ called.number_calling_me += 1
1235+
1236+ def _rest_node_output_to_graph(self, rest_output):
1237+ """
1238+ Convert the output of a REST API query into our internal representation.
1239+ :param rest_output: output of the REST call as a Neo4j QuerySequence
1240+ :return: iterable of <Function>s ready to initialise self.functions.
1241+ """
1242+
1243+ if rest_output is None or not rest_output.elements:
1244+ return []
1245+
1246+ # how we store this is: a dict
1247+ # with keys 'functionname'
1248+ # and values [the function object we will use,
1249+ # and a set of (function names this function calls),
1250+ # and numeric ID of this node in the Neo4J database]
1251+
1252+ result = {}
1253+
1254+ # initial pass for names of functions
1255+
1256+ # if the following assertion failed, we've probably called db.query
1257+ # to get it to not return client.Node objects, which is wrong.
1258+ # we attempt to handle this a bit later; this should never arise, but
1259+ # we can cope with it happening in some cases, like the test suite
1260+
1261+ if type(rest_output.elements) is not list:
1262+ logging.warning('Not a list: {}'.format(type(rest_output.elements)))
1263+
1264+ for node_list in rest_output.elements:
1265+ assert(isinstance(node_list, list))
1266+ for node in node_list:
1267+ if isinstance(node, client.Node):
1268+ name = node.properties['name']
1269+ node_id = node.id
1270+ node_type = node.properties['type']
1271+ else: # this is the handling we mentioned earlier;
1272+ # we are a dictionary instead of a list, as for some
1273+ # reason we've returned Raw rather than Node data.
1274+ # We should never reach this code, but just in case.
1275+ name = node['data']['name']
1276+ # hacky workaround to get the id
1277+ node_id = node['self'].split('/')[-1]
1278+ node_type = node['data']['type']
1279+
1280+ result[name] = [Function(self.program_name,
1281+ function_name=name,
1282+ function_type=node_type),
1283+ set(),
1284+ node_id]
1285+
1286+ # end initialisation of names-dictionary
1287+
1288+ if self._parent_db_connection is not None:
1289+ # This is the normal case, of extracting results from a server.
1290+ # We leave the other case in because it is useful for unit testing.
1291+
1292+ # We collect the name-name pairs of caller-callee, batched for speed
1293+ new_tx = self._parent_db_connection.transaction(using_globals=False,
1294+ for_query=True)
1295+ for index in result:
1296+ q = ("START n=node({})"
1297+ "MATCH n-[calls:calls]->(m)"
1298+ "RETURN n.name, m.name").format(result[index][2])
1299+ new_tx.append(q)
1300+
1301+ logging.debug('exec')
1302+ results = new_tx.execute()
1303+
1304+ # results is a list of query results, each of those being a list of
1305+ # calls.
1306+
1307+ for call_list in results:
1308+ if call_list:
1309+ # call_list has element 0 being an arbitrary call this
1310+ # function makes; element 0 of that call is the name of the
1311+ # function itself. Think {{'orig', 'b'}, {'orig', 'c'}}.
1312+ orig = call_list[0][0]
1313+ # result['orig'] is [<Function>, ('callee1','callee2')]
1314+ result[orig][1] |= set(list(zip(*call_list.elements))[1])
1315+ # recall: set union is denoted by |
1316+
1317+ else:
1318+ # we don't have a parent database connection.
1319+ # This has probably arisen because we created this object from a
1320+ # test suite, or something like that.
1321+ for node in rest_output.elements:
1322+ node_name = node[0].properties['name']
1323+ result[node_name][1] |= {relationship.end.properties['name']
1324+ for relationship in node[0].relationships.outgoing()}
1325+
1326+ logging.debug('Relationships complete.')
1327+
1328+ # named_function takes a function name and returns the Function object
1329+ # with that name, or None if none exists.
1330+ named_function = lambda name: result[name][0] if name in result else None
1331+
1332+ for function, calls, node_id in result.values():
1333+ what_i_call = [named_function(name)
1334+ for name in calls
1335+ if named_function(name) is not None]
1336+ function.functions_i_call = what_i_call
1337+
1338+ return [list_element[0]
1339+ for list_element in result.values()
1340+ if list_element[0]]
1341+
1342+ def get_functions(self):
1343+ """
1344+ :return: a list of Function objects present in the query result
1345+ """
1346+ return self.functions
1347+
1348+ def get_function(self, name):
1349+ """
1350+ Given a function name, returns the Function object which has that name.
1351+ If no function with that name exists, returns None.
1352+ """
1353+ func_list = [func for func in self.functions if func.name == name]
1354+ return None if len(func_list) == 0 else func_list[0]
1355+
1356+
1357+def set_common_cutoff(common_def):
1358+ """
1359+ Sets the number of incoming connections at which we deem a function 'common'
1360+ Default is 10 (which is used if this method is never called).
1361+ :param common_def: number of incoming connections
1362+ """
1363+ global COMMON_CUTOFF
1364+ COMMON_CUTOFF = common_def
1365+
1366+
1367+class Function(object):
1368+ """Represents a function which might appear in a FunctionQueryResult."""
1369+
1370+ def __eq__(self, other):
1371+ funcs_i_call_list = {func.name for func in self.functions_i_call}
1372+ funcs_other_calls_list = {func.name for func in other.functions_i_call}
1373+
1374+ return (self.parent_program == other.parent_program
1375+ and self.name == other.name
1376+ and funcs_i_call_list == funcs_other_calls_list
1377+ and self.attributes == other.attributes)
1378+
1379+ @property
1380+ def number_calling_me(self):
1381+ return self._number_calling_me
1382+
1383+ @number_calling_me.setter
1384+ def number_calling_me(self, value):
1385+ self._number_calling_me = value
1386+ self.is_common = (self._number_calling_me > COMMON_CUTOFF)
1387+
1388+ def __init__(self, program_name='', function_name='', function_type=''):
1389+ self.parent_program = program_name
1390+ self.attributes = []
1391+ self.type = function_type
1392+ self.functions_i_call = []
1393+ self.name = function_name
1394+ self.is_common = False
1395+ self._number_calling_me = 0
1396+ # care: _number_calling_me is not automatically updated, except by
1397+ # any invocation of FunctionQueryResult._update_common_functions.
1398+
1399+
1400+class SextantConnection:
1401+ """
1402+ RESTful connection to a remote database.
1403+ It can be used to create/delete/query programs.
1404+ """
1405+
1406+ def __init__(self, url):
1407+ self.url = url
1408+ self._db = GraphDatabase(url)
1409+
1410+ def new_program(self, name_of_program):
1411+ """
1412+ Request that the remote database create a new program with the given name.
1413+ This procedure will create a new program remotely; you can manipulate
1414+ that program using the returned AddToDatabase object.
1415+ The name can appear in the database already, but this is not recommended
1416+ because then delete_program will not know which to delete. Check first
1417+ using self.check_program_exists.
1418+ The name specified must pass Validator.validate()ion; this is a measure
1419+ to prevent Cypher injection attacks.
1420+ :param name_of_program: string program name
1421+ :return: AddToDatabase instance if successful
1422+ """
1423+
1424+ if not Validator.validate(name_of_program):
1425+ raise ValueError(
1426+ "{} is not a valid program name".format(name_of_program))
1427+
1428+ return AddToDatabase(sextant_connection=self,
1429+ program_name=name_of_program)
1430+
1431+ def delete_program(self, name_of_program):
1432+ """
1433+ Request that the remote database delete a specified program.
1434+ :param name_of_program: a string which must be alphanumeric only
1435+ :return: bool(request succeeded)
1436+ """
1437+ if not Validator.validate(name_of_program):
1438+ return False
1439+
1440+ q = """MATCH (n) WHERE n.name= "{}" AND n.type="program"
1441+ OPTIONAL MATCH (n)-[r]-(b) OPTIONAL MATCH (b)-[rel]-()
1442+ DELETE b,rel DELETE n, r""".format(name_of_program)
1443+
1444+ self._db.query(q)
1445+
1446+ return True
1447+
1448+ def _execute_query(self, prog_name='', query=''):
1449+ """
1450+ Executes a Cypher query against the remote database.
1451+ Note that this returns a FunctionQueryResult, so is unsuitable for any
1452+ other expected outputs (such as lists of names). For those instances,
1453+ it is better to run self._parent_database_connection_object.query
1454+ explicitly.
1455+ Intended only to be used for non-updating queries
1456+ (such as "get functions" rather than "create").
1457+ :param prog_name: name of the program the result object will reflect
1458+ :param query: verbatim query we wish the server to execute
1459+ :return: a FunctionQueryResult corresponding to the server's output
1460+ """
1461+ rest_output = self._db.query(query, returns=client.Node)
1462+
1463+ return FunctionQueryResult(parent_db=self._db,
1464+ program_name=prog_name,
1465+ rest_output=rest_output)
1466+
1467+ def get_program_names(self):
1468+ """
1469+ Execute query to retrieve a list of all programs in the database.
1470+ Any name in this list can be used verbatim in any SextantConnection
1471+ method which requires a program-name input.
1472+ :return: a list of function-name strings.
1473+ """
1474+ q = """MATCH (n) WHERE n.type = "program" RETURN n.name"""
1475+ program_names = self._db.query(q, returns=str).elements
1476+
1477+ result = [el[0] for el in program_names]
1478+
1479+ return set(result)
1480+
1481+ def check_program_exists(self, program_name):
1482+ """
1483+ Execute query to check whether a program with the given name exists.
1484+ Returns False if the program_name fails validation against Validator.
1485+ :return: bool(the program exists in the database).
1486+ """
1487+
1488+ if not Validator.validate(program_name):
1489+ return False
1490+
1491+ q = ("MATCH (base) WHERE base.name = '{}' AND base.type = 'program' "
1492+ "RETURN count(base)").format(program_name)
1493+
1494+ result = self._db.query(q, returns=int)
1495+ return result.elements[0][0] > 0
1496+
1497+ def check_function_exists(self, program_name, function_name):
1498+ """
1499+ Execute query to check whether a function with the given name exists.
1500+ We only check for functions which are children of a program with the
1501+ given program_name.
1502+ :param program_name: string name of the program within which to check
1503+ :param function_name: string name of the function to check for existence
1504+ :return: bool(names validate correctly, and function exists in program)
1505+ """
1506+ if not self.check_program_exists(program_name):
1507+ return False
1508+
1509+ if not Validator.validate(program_name):
1510+ return False
1511+
1512+ q = ("MATCH (base) WHERE base.name = '{}' AND base.type = 'program'"
1513+ "MATCH (base)-[r:subject]->(m) WHERE m.name = '{}'"
1514+ "RETURN count(m)").format(program_name, function_name)
1515+
1516+ result = self._db.query(q, returns=int)
1517+ return result.elements[0][0] > 0
1518+
1519+ def get_function_names(self, program_name):
1520+ """
1521+ Execute query to retrieve a list of all functions in the program.
1522+ Any of the output names can be used verbatim in any SextantConnection
1523+ method which requires a function-name input.
1524+ :param program_name: name of the program whose functions to retrieve
1525+ :return: None if program_name doesn't exist in the remote database,
1526+ a set of function-name strings otherwise.
1527+ """
1528+
1529+ if not self.check_program_exists(program_name):
1530+ return None
1531+
1532+ q = ("MATCH (base) WHERE base.name = '{}' AND base.type = 'program' "
1533+ "MATCH (base)-[r:subject]->(m) "
1534+ "RETURN m.name").format(program_name)
1535+ return {func[0] for func in self._db.query(q)}
1536+
1537+ def get_all_functions_called(self, program_name, function_calling):
1538+ """
1539+ Execute query to find all functions called by a function (indirectly).
1540+ If the given function is not present in the program, returns None;
1541+ likewise if the program_name does not exist.
1542+ :param program_name: a string name of the program we wish to query under
1543+ :param function_calling: string name of a function whose children to find
1544+ :return: FunctionQueryResult, maximal subgraph rooted at function_calling
1545+ """
1546+
1547+ if not self.check_program_exists(program_name):
1548+ return None
1549+
1550+ if not self.check_function_exists(program_name, function_calling):
1551+ return None
1552+
1553+ q = """MATCH (base) WHERE base.name = '{}' ANd base.type = 'program'
1554+ MATCH (base)-[:subject]->(m) WHERE m.name='{}'
1555+ MATCH (m)-[:calls*]->(n)
1556+ RETURN distinct n, m""".format(program_name, function_calling)
1557+
1558+ return self._execute_query(program_name, q)
1559+
1560+ def get_all_functions_calling(self, program_name, function_called):
1561+ """
1562+ Execute query to find all functions which call a function (indirectly).
1563+ If the given function is not present in the program, returns None;
1564+ likewise if the program_name does not exist.
1565+ :param program_name: a string name of the program we wish to query
1566+ :param function_called: string name of a function whose parents to find
1567+ :return: FunctionQueryResult, maximal connected subgraph with leaf function_called
1568+ """
1569+
1570+ if not self.check_program_exists(program_name):
1571+ return None
1572+
1573+ if not self.check_function_exists(program_name, function_called):
1574+ return None
1575+
1576+ q = """MATCH (base) WHERE base.name = '{}' AND base.type = 'program'
1577+ MATCH (base)-[r:subject]->(m) WHERE m.name='{}'
1578+ MATCH (n)-[:calls*]->(m) WHERE n.name <> '{}'
1579+ RETURN distinct n , m"""
1580+ q = q.format(program_name, function_called, program_name)
1581+
1582+ return self._execute_query(program_name, q)
1583+
1584+ def get_call_paths(self, program_name, function_calling, function_called):
1585+ """
1586+ Execute query to find all possible routes between two specific nodes.
1587+ If the given functions are not present in the program, returns None;
1588+ ditto if the program_name does not exist.
1589+ :param program_name: string program name
1590+ :param function_calling: string
1591+ :param function_called: string
1592+ :return: FunctionQueryResult, the union of all subgraphs reachable by
1593+ adding a source at function_calling and a sink at function_called.
1594+ """
1595+
1596+ if not self.check_program_exists(program_name):
1597+ return None
1598+
1599+ if not self.check_function_exists(program_name, function_called):
1600+ return None
1601+
1602+ if not self.check_function_exists(program_name, function_calling):
1603+ return None
1604+
1605+ q = r"""MATCH (pr) WHERE pr.name = '{}' AND pr.type = 'program'
1606+ MATCH p=(start {{name: "{}" }})-[:calls*]->(end {{name:"{}"}})
1607+ WHERE (pr)-[:subject]->(start)
1608+ WITH DISTINCT nodes(p) AS result
1609+ UNWIND result AS answer
1610+ RETURN answer"""
1611+ q = q.format(program_name, function_calling, function_called)
1612+
1613+ return self._execute_query(program_name, q)
1614+
1615+ def get_whole_program(self, program_name):
1616+ """Execute query to find the entire program with a given name.
1617+ If the program is not present in the remote database, returns None.
1618+ :param: program_name: a string name of the program we wish to return.
1619+ :return: a FunctionQueryResult consisting of the program graph.
1620+ """
1621+
1622+ if not self.check_program_exists(program_name):
1623+ return None
1624+
1625+ query = """MATCH (base) WHERE base.name = '{}' AND base.type = 'program'
1626+ MATCH (base)-[subject:subject]->(m)
1627+ RETURN DISTINCT (m)""".format(program_name)
1628+
1629+ return self._execute_query(program_name, query)
1630+
1631+ def get_shortest_path_between_functions(self, program_name, func1, func2):
1632+ """
1633+ Execute query to get a single, shortest, path between two functions.
1634+ :param program_name: string name of the program we wish to search under
1635+ :param func1: the name of the originating function of our shortest path
1636+ :param func2: the name of the function at which to terminate the path
1637+ :return: FunctionQueryResult shortest path between func1 and func2.
1638+ """
1639+ if not self.check_program_exists(program_name):
1640+ return None
1641+
1642+ if not self.check_function_exists(program_name, func1):
1643+ return None
1644+
1645+ if not self.check_function_exists(program_name, func2):
1646+ return None
1647+
1648+ q = """MATCH (func1 {{ name:"{}" }}),(func2 {{ name:"{}" }}),
1649+ p = shortestPath((func1)-[:calls*]->(func2))
1650+ UNWIND nodes(p) AS ans
1651+ RETURN ans""".format(func1, func2)
1652+
1653+ return self._execute_query(program_name, q)
1654
1655=== added file 'src/sextant/errors.py'
1656--- src/sextant/errors.py 1970-01-01 00:00:00 +0000
1657+++ src/sextant/errors.py 2014-08-15 12:04:08 +0000
1658@@ -0,0 +1,20 @@
1659+# -----------------------------------------------------------------------------
1660+# errors.py -- Sextant error definitions
1661+#
1662+# August 2014, Phil Connell
1663+#
1664+# Copyright 2014, Ensoft Ltd.
1665+# -----------------------------------------------------------------------------
1666+
1667+from __future__ import absolute_import, print_function
1668+
1669+__all__ = (
1670+ "MissingDependencyError",
1671+)
1672+
1673+
1674+class MissingDependencyError(Exception):
1675+ """
1676+ Raised if trying an operation for which an optional dependency is missing.
1677+ """
1678+
1679
1680=== added file 'src/sextant/export.py'
1681--- src/sextant/export.py 1970-01-01 00:00:00 +0000
1682+++ src/sextant/export.py 2014-08-15 12:04:08 +0000
1683@@ -0,0 +1,152 @@
1684+# -----------------------------------------
1685+# Sextant
1686+# Copyright 2014, Ensoft Ltd.
1687+# Author: Patrick Stevens, James Harkin
1688+# -----------------------------------------
1689+# Convert a program from internal Python representation to various output formats
1690+
1691+__all__ = "ProgramConverter"
1692+
1693+
1694+class ProgramConverter:
1695+ # Given our internal program representation, converts it to output formats.
1696+ # Currently supported: GraphML
1697+
1698+ @staticmethod
1699+ def get_supported_outputs():
1700+ return ['graphml', 'yed_graphml', 'dot']
1701+
1702+ @staticmethod
1703+ def get_display_name(function, suppress_common_nodes=False):
1704+ """
1705+ Given a Function object, retrieve the label we attach to it.
1706+ For instance, function-pointers are labelled "(function pointer)",
1707+ while the `main` function is usually labelled "main".
1708+ """
1709+ if function.type == "function_pointer":
1710+ name = "(function pointer)"
1711+ else:
1712+ name = function.name
1713+
1714+ if suppress_common_nodes and function.is_common:
1715+ name += ' (common)'
1716+ return name
1717+
1718+ @staticmethod
1719+ def to_dot(program, suppress_common_nodes=False):
1720+ """
1721+ Convert the program to DOT output format.
1722+ """
1723+ output_str = 'digraph "{}" {{\n '.format(program.program_name)
1724+ output_str += 'overlap=false; \n'
1725+
1726+ font_name = "helvetica"
1727+
1728+ for func in program.get_functions():
1729+ if func.type == "plt_stub":
1730+ output_str += ' "{}" [fillcolor=pink, style=filled]\n'.format(func.name)
1731+ elif func.type == "function_pointer":
1732+ output_str += ' "{}" [fillcolor=yellow, style=filled]\n'.format(func.name)
1733+
1734+ # in all cases, even if we've specified that we want a filled-in
1735+ # node already, DOT lets us add more information about that node
1736+ # so we can insist on turning that same node into a box-shape
1737+ # and changing its font.
1738+ output_str += ' "{}" [label="{}", fontname="{}", shape=box]\n'.format(func.name,
1739+ ProgramConverter.get_display_name(func, suppress_common_nodes),
1740+ font_name)
1741+ if func.is_common:
1742+ output_str += ' "{}" [fillcolor=lightgreen, style=filled]\n'.format(func.name)
1743+
1744+ for func_called in func.functions_i_call:
1745+ if not (suppress_common_nodes and func_called.is_common):
1746+ output_str += ' "{}" -> "{}"\n'.format(func.name, func_called.name)
1747+
1748+ output_str += '}'
1749+ return output_str
1750+
1751+
1752+
1753+ @staticmethod
1754+ def to_yed_graphml(program, suppress_common_nodes=False):
1755+ commonly_called = []
1756+ output_str = """<?xml version="1.0" encoding="UTF-8" standalone="no"?>
1757+<graphml xmlns="http://graphml.graphdrawing.org/xmlns" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" \
1758+xmlns:y="http://www.yworks.com/xml/graphml" xmlns:yed="http://www.yworks.com/xml/yed/3" \
1759+xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://www.yworks.com/xml/schema/graphml/1.1/ygraphml.xsd">
1760+"""
1761+ output_str += """<key for="graphml" id="d0" yfiles.type="resources"/>
1762+ <key for="port" id="d1" yfiles.type="portgraphics"/>
1763+ <key for="port" id="d2" yfiles.type="portgeometry"/>
1764+ <key for="port" id="d3" yfiles.type="portuserdata"/>
1765+ <key attr.name="url" attr.type="string" for="node" id="d4"/>
1766+ <key attr.name="description" attr.type="string" for="node" id="d5"/>
1767+ <key for="node" id="d6" yfiles.type="nodegraphics"/>
1768+ <key attr.name="url" attr.type="string" for="edge" id="d7"/>
1769+ <key attr.name="description" attr.type="string" for="edge" id="d8"/>
1770+ <key for="edge" id="d9" yfiles.type="edgegraphics"/>\n"""
1771+
1772+ output_str += """<graph id="{}" edgedefault="directed">\n""".format(program.program_name)
1773+
1774+ for func in program.get_functions():
1775+ display_func = ProgramConverter.get_display_name(func)
1776+ if func.type == "plt_stub":
1777+ colour = "#ff00ff"
1778+ elif func.type == "function_pointer":
1779+ colour = "#99ffff"
1780+ elif func.is_common:
1781+ colour = "#00FF00"
1782+ else:
1783+ colour = "#ffcc00"
1784+ output_str += """<node id="{}">
1785+ <data key="d6">
1786+ <y:ShapeNode>
1787+ <y:Geometry height="{}" width="{}" x="60.0" y="0.0"/>
1788+ <y:Fill color="{}" transparent="false"/>
1789+ <y:BorderStyle color="#000000" type="line" width="1.0"/>
1790+ <y:NodeLabel alignment="center" autoSizePolicy="content" fontFamily="Dialog"
1791+ fontSize="12" fontStyle="plain" hasBackgroundColor="false" hasLineColor="false"
1792+ height="18.701171875" modelName="custom" textColor="#000000" visible="true"
1793+ width="36.6953125" x="-3.34765625" y="5.6494140625">{}<y:LabelModel>
1794+ <y:SmartNodeLabelModel distance="4.0"/>
1795+ </y:LabelModel>
1796+ <y:ModelParameter>
1797+ <y:SmartNodeLabelModelParameter labelRatioX="0.0" labelRatioY="0.0"
1798+ nodeRatioX="0.0" nodeRatioY="0.0" offsetX="0.0" offsetY="0.0" upX="0.0"
1799+ upY="-1.0"/>
1800+ </y:ModelParameter>
1801+ </y:NodeLabel>
1802+ <y:Shape type="rectangle"/>
1803+ </y:ShapeNode>
1804+ </data>
1805+ </node>\n""".format(func.name, 20, len(display_func)*8, colour, display_func)
1806+ for callee in func.functions_i_call:
1807+ if callee not in commonly_called:
1808+ if not(suppress_common_nodes and callee.is_common):
1809+ output_str += """<edge source="{}" target="{}"> <data key="d9">
1810+ <y:PolyLineEdge>
1811+ <y:LineStyle color="#000000" type="line" width="1.0"/>
1812+ <y:Arrows source="none" target="standard"/>
1813+ <y:BendStyle smoothed="false"/>
1814+ </y:PolyLineEdge>
1815+ </data>
1816+ </edge>\n""".format(func.name, callee.name)
1817+
1818+ output_str += '</graph>\n<data key="d0"> <y:Resources/> </data>\n</graphml>'
1819+ return output_str
1820+
1821+ @staticmethod
1822+ def to_graphml(program):
1823+ output_str = """<?xml version="1.0" encoding="UTF-8"?>
1824+<graphml xmlns="http://graphml.graphdrawing.org/xmlns" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" \
1825+xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">
1826+"""
1827+ output_str += '<graph id="{}" edgedefault="directed">\n'.format(program.program_name)
1828+
1829+ for func in program.get_functions():
1830+ output_str += '<node id="{}"> <data key="name">{}</data> </node>\n'.format(func.name, ProgramConverter.get_display_name(func))
1831+ for callee in func.functions_i_call:
1832+ output_str += '<edge source="{}" target="{}"> <data key="calls">1</data> </edge>\n'.format(func.name, callee.name)
1833+
1834+ output_str += '</graph>\n</graphml>'
1835+ return output_str
1836\ No newline at end of file
1837
1838=== added file 'src/sextant/objdump_parser.py'
1839--- src/sextant/objdump_parser.py 1970-01-01 00:00:00 +0000
1840+++ src/sextant/objdump_parser.py 2014-08-15 12:04:08 +0000
1841@@ -0,0 +1,272 @@
1842+# -----------------------------------------
1843+# Sextant
1844+# Copyright 2014, Ensoft Ltd.
1845+# Author: Patrick Stevens
1846+# -----------------------------------------
1847+
1848+#!/usr/bin/python3
1849+
1850+import re
1851+import argparse
1852+import os.path
1853+import subprocess
1854+import logging
1855+
1856+
1857+class ParsedObject():
1858+ """
1859+ Represents a function as parsed from an objdump disassembly.
1860+ Has a name (which is the verbatim name like '__libc_start_main@plt'),
1861+ a position (which is the virtual memory location in hex, like '08048320'
1862+ extracted from the dump),
1863+ and a canonical_position (which is the virtual memory location in hex
1864+ but stripped of leading 0s, so it should be a
1865+ unique id).
1866+ It also has a list what_do_i_call of ParsedObjects it calls using the
1867+ assembly keyword 'call'.
1868+ It has a list original_code of its assembler code, too, in case it's useful.
1869+ """
1870+
1871+ @staticmethod
1872+ def get_canonical_position(position):
1873+ return position.lstrip('0')
1874+
1875+ def __eq__(self, other):
1876+ return self.name == other.name
1877+
1878+ def __init__(self, input_lines=None, assembler_section='', function_name='',
1879+ ignore_function_pointers=True, function_pointer_id=None):
1880+ """
1881+ Create a new ParsedObject given the definition-lines from objdump -S.
1882+ A sample first definition-line is '08048300 <__gmon_start__@plt>:\n'
1883+ but this method
1884+ expects to see the entire definition eg
1885+
1886+080482f0 <puts@plt>:
1887+ 80482f0: ff 25 00 a0 04 08 jmp *0x804a000
1888+ 80482f6: 68 00 00 00 00 push $0x0
1889+ 80482fb: e9 e0 ff ff ff jmp 80482e0 <_init+0x30>
1890+
1891+ We also might expect assembler_section, which is for instance '.init'
1892+ in 'Disassembly of section .init:'
1893+ function_name is used if we want to give this function a custom name.
1894+ ignore_function_pointers=True will pretend that calls to (eg) *eax do
1895+ not exist; setting to False makes us create stubs for those calls.
1896+ function_pointer_id is only used internally; it refers to labelling
1897+ of function pointers if ignore_function_pointers is False. Each
1898+ stub is given a unique numeric ID: this parameter tells init where
1899+ to start counting these IDs from.
1900+
1901+ """
1902+ if input_lines is None:
1903+ # get around Python's inability to pass in empty lists by value
1904+ input_lines = []
1905+
1906+ self.name = function_name or re.search(r'<.+>', input_lines[0]).group(0).strip('<>')
1907+ self.what_do_i_call = []
1908+ self.position = ''
1909+
1910+ if input_lines:
1911+ self.position = re.search(r'^[0-9a-f]+', input_lines[0]).group(0)
1912+ self.canonical_position = ParsedObject.get_canonical_position(self.position)
1913+ self.assembler_section = assembler_section
1914+ self.original_code = input_lines[1:]
1915+
1916+ call_regex_compiled = (ignore_function_pointers and re.compile(r'\tcall. +[^\*]+\n')) or re.compile(r'\tcall. +.+\n')
1917+
1918+ lines_where_i_call = [line for line in input_lines if call_regex_compiled.search(line)]
1919+
1920+ if not ignore_function_pointers and not function_pointer_id:
1921+ function_pointer_id = [1]
1922+
1923+ for line in lines_where_i_call:
1924+ # we'll catch call and callq for the moment
1925+ called = (call_regex_compiled.search(line).group(0))[8:].lstrip(' ').rstrip('\n')
1926+ if called[0] == '*' and ignore_function_pointers == False:
1927+ # we have a function pointer, which we'll want to give a distinct name
1928+ address = '0'
1929+ name = '_._function_pointer_' + str(function_pointer_id[0])
1930+ function_pointer_id[0] += 1
1931+
1932+ self.what_do_i_call.append((address, name))
1933+
1934+ else: # we're not on a function pointer
1935+ called_split = called.split(' ')
1936+ if len(called_split) == 2:
1937+ address, name = called_split
1938+ name = name.strip('<>')
1939+ # we still want to remove address offsets like +0x09 from the end of name
1940+ match = re.match(r'^.+(?=\+0x[a-f0-9]+$)', name)
1941+ if match is not None:
1942+ name = match.group(0)
1943+ self.what_do_i_call.append((address, name.strip('<>')))
1944+ else: # the format of the "what do i call" is not recognised as a name/address pair
1945+ self.what_do_i_call.append(tuple(called_split))
1946+
1947+ def __str__(self):
1948+ if self.position:
1949+ return 'Memory address ' + self.position + ' with name ' + self.name + ' in section ' + str(
1950+ self.assembler_section)
1951+ else:
1952+ return 'Name ' + self.name
1953+
1954+ def __repr__(self):
1955+ out_str = 'Disassembly of section ' + self.assembler_section + ':\n\n' + self.position + ' ' + self.name + ':\n'
1956+ return out_str + '\n'.join([' ' + line for line in self.original_code])
1957+
1958+
1959+class Parser:
1960+ # Class to manipulate the output of objdump
1961+
1962+ def __init__(self, input_file_location='', file_contents=None, sections_to_view=None, ignore_function_pointers=False):
1963+ """Creates a new Parser, given an input file path. That path should be an output from objdump -D.
1964+ Alternatively, supply file_contents, as a list of each line of the objdump output. We expect newlines
1965+ to have been stripped from the end of each of these lines.
1966+ sections_to_view makes sure we only use the specified sections (use [] for 'all sections' and None for none).
1967+ """
1968+ if file_contents is None:
1969+ file_contents = []
1970+
1971+ if sections_to_view is None:
1972+ sections_to_view = []
1973+
1974+ if input_file_location:
1975+ file_to_read = open(input_file_location, 'r')
1976+ self.source_string_list = [line for line in file_to_read]
1977+ file_to_read.close()
1978+ elif file_contents:
1979+ self.source_string_list = [string + '\n' for string in file_contents]
1980+ self.parsed_objects = []
1981+ self.sections_to_view = sections_to_view
1982+ self.ignore_function_pointers = ignore_function_pointers
1983+ self.pointer_identifier = [1]
1984+
1985+ def create_objects(self):
1986+ """ Go through the source_string_list, getting object names (like 'main') along with the corresponding
1987+ definitions, and put them into parsed_objects """
1988+ if self.sections_to_view is None:
1989+ return
1990+
1991+ is_in_section = lambda name: self.sections_to_view == [] or name in self.sections_to_view
1992+
1993+ parsed_objects = []
1994+ current_object = []
1995+ current_section = ''
1996+ regex_compiled_addr_and_name = re.compile(r'[0-9a-f]+ <.+>:\n')
1997+ regex_compiled_section = re.compile(r'section .+:\n')
1998+
1999+ for line in self.source_string_list[4:]: # we bodge, since the file starts with a little bit of guff
2000+ if regex_compiled_addr_and_name.match(line):
2001+ # we are a starting line
2002+ current_object = [line]
2003+ elif re.match(r'Disassembly of section', line):
2004+ current_section = regex_compiled_section.search(line).group(0).lstrip('section ').rstrip(':\n')
2005+ current_object = []
2006+ elif line == '\n':
2007+ # we now need to stop parsing the current block, and store it
2008+ if len(current_object) > 0 and is_in_section(current_section):
2009+ parsed_objects.append(ParsedObject(input_lines=current_object, assembler_section=current_section,
2010+ ignore_function_pointers=self.ignore_function_pointers,
2011+ function_pointer_id=self.pointer_identifier))
2012+ else:
2013+ current_object.append(line)
2014+
2015+ # now we should be done. We assumed that blocks begin with r'[0-9a-f]+ <.+>:\n' and end with a newline.
2016+ # clear duplicates:
2017+
2018+ self.parsed_objects = []
2019+ for obj in parsed_objects:
2020+ if obj not in self.parsed_objects: # this is so that if we jump into the function at an offset,
2021+ # we still register it as being the old function, not some new function at a different address
2022+ # with the same name
2023+ self.parsed_objects.append(obj)
2024+
2025+ # by this point, each object contains a self.what_do_i_call which is a list of tuples
2026+ # ('address', 'name') if the address and name were recognised, or else (thing1, thing2, ...)
2027+ # where the instruction was call thing1 thing2 thing3... .
2028+
2029+ def object_lookup(self, object_name='', object_address=''):
2030+ """Returns the object with name object_name or address object_address (at least one must be given).
2031+ If objects with the given name or address
2032+ are not found, returns None."""
2033+
2034+ if object_name == '' and object_address == '':
2035+ return None
2036+
2037+ trial_obj = self.parsed_objects
2038+
2039+ if object_name != '':
2040+ trial_obj = [obj for obj in trial_obj if obj.name == object_name]
2041+
2042+ if object_address != '':
2043+ trial_obj = [obj for obj in trial_obj if
2044+ obj.canonical_position == ParsedObject.get_canonical_position(object_address)]
2045+
2046+ if len(trial_obj) == 0:
2047+ return None
2048+
2049+ return trial_obj
2050+
2051+def get_parsed_objects(filepath, sections_to_view, not_object_file, readable=False, ignore_function_pointers=False):
2052+ if sections_to_view is None:
2053+ sections_to_view = [] # because we use None for "no sections"; the intent of not providing any sections
2054+ # on the command line was to look at all sections, not none
2055+
2056+ # first, check whether the given file exists
2057+ if not os.path.isfile(filepath):
2058+ logging.error('Input file does not exist')
2059+ return False
2060+
2061+ #now the file should exist
2062+ if not not_object_file: #if it is something we need to run through objdump first
2063+ #we need first to run the object file through objdump
2064+
2065+ objdump_file_contents = subprocess.check_output(['objdump', '-D', filepath])
2066+ objdump_str = objdump_file_contents.decode('utf-8')
2067+
2068+ p = Parser(file_contents=objdump_str.split('\n'), sections_to_view=sections_to_view, ignore_function_pointers=ignore_function_pointers)
2069+ else:
2070+ try:
2071+ p = Parser(input_file_location=filepath, sections_to_view=sections_to_view, ignore_function_pointers=ignore_function_pointers)
2072+ except UnicodeDecodeError:
2073+ logging.error('File could not be parsed as a string. Did you mean to supply --object-file?')
2074+ return False
2075+
2076+ if readable: # if we're being called from the command line
2077+ print('File read; beginning parse.')
2078+ #file is now read, and we start parsing
2079+
2080+ p.create_objects()
2081+ return p.parsed_objects
2082+
2083+def main():
2084+ argumentparser = argparse.ArgumentParser(description="Parse the output of objdump.")
2085+ argumentparser.add_argument('--filepath', metavar="FILEPATH", help="path to input file", type=str, nargs=1)
2086+ argumentparser.add_argument('--not-object-file', help="import text objdump output instead of the compiled file", default=False,
2087+ action='store_true')
2088+ argumentparser.add_argument('--sections-to-view', metavar="SECTIONS",
2089+ help="sections of disassembly to view, like '.text'; leave blank for 'all'",
2090+ type=str, nargs='*')
2091+ argumentparser.add_argument('--ignore-function-pointers', help='whether to skip parsing calls to function pointers', action='store_true', default=False)
2092+
2093+ parsed = argumentparser.parse_args()
2094+
2095+ filepath = parsed.filepath[0]
2096+ sections_to_view = parsed.sections_to_view
2097+ not_object_file = parsed.not_object_file
2098+ readable = True
2099+ function_pointers = parsed.ignore_function_pointers
2100+
2101+ parsed_objs = get_parsed_objects(filepath, sections_to_view, not_object_file, readable, function_pointers)
2102+ if parsed_objs is False:
2103+ return 1
2104+
2105+ if readable:
2106+ for named_function in parsed_objs:
2107+ print(named_function.name)
2108+ print([f[-1] for f in named_function.what_do_i_call]) # use [-1] to get the last element, since:
2109+ #either we are in ('address', 'name'), when we want the last element, or else we are in (thing1, thing2, ...)
2110+ #so for the sake of argument we'll take the last thing
2111+
2112+if __name__ == "__main__":
2113+ main()
2114
2115=== added file 'src/sextant/pyinput.py'
2116--- src/sextant/pyinput.py 1970-01-01 00:00:00 +0000
2117+++ src/sextant/pyinput.py 2014-08-15 12:04:08 +0000
2118@@ -0,0 +1,180 @@
2119+# -----------------------------------------------------------------------------
2120+# pyinput.py -- Input information from Python programs.
2121+#
2122+# August 2014, Phil Connell
2123+#
2124+# Copyright 2014, Ensoft Ltd.
2125+# -----------------------------------------------------------------------------
2126+
2127+from __future__ import absolute_import, print_function
2128+
2129+__all__ = (
2130+ "trace",
2131+)
2132+
2133+
2134+import contextlib
2135+import sys
2136+
2137+from . import errors
2138+from . import db_api
2139+
2140+
2141+# Optional, should be checked at API entrypoints requiring entrails (and
2142+# yes, the handling is a bit fugly).
2143+try:
2144+ import entrails
2145+except ImportError:
2146+ _entrails_available = False
2147+ class entrails:
2148+ EntrailsOutput = object
2149+else:
2150+ _entrails_available = True
2151+
2152+
2153+class _SextantOutput(entrails.EntrailsOutput):
2154+ """Record calls traced by entrails in a sextant database."""
2155+
2156+ # Internal attributes:
2157+ #
2158+ # _conn:
2159+ # Sextant connection.
2160+ # _fns:
2161+ # Stack of function names (implemented as a list), reflecting the current
2162+ # call stack, based on enter, exception and exit events.
2163+ # _prog:
2164+ # Sextant program representation.
2165+ _conn = None
2166+ _fns = None
2167+ _prog = None
2168+
2169+ def __init__(self, conn, program_name):
2170+ """
2171+ Initialise this output.
2172+
2173+ conn:
2174+ Connection to the Sextant database.
2175+ program_name:
2176+ String used to refer to the traced program in sextant.
2177+
2178+ """
2179+ self._conn = conn
2180+ self._fns = []
2181+ self._prog = self._conn.new_program(program_name)
2182+ self._tracer = self._trace()
2183+ next(self._tracer)
2184+
2185+ def _add_frame(self, event):
2186+ """Add a function call to the internal stack."""
2187+ name = event.qualname()
2188+ self._fns.append(name)
2189+ self._prog.add_function(name)
2190+
2191+ try:
2192+ prev_name = self._fns[-2]
2193+ except IndexError:
2194+ pass
2195+ else:
2196+ self._prog.add_function_call(prev_name, name)
2197+
2198+ def _remove_frame(self, event):
2199+ """Remove a function call from the internal stack."""
2200+ assert event.qualname() == self._fns[-1], \
2201+ "Unexpected event for {}".format(event.qualname())
2202+ self._fns.pop()
2203+
2204+ def _handle_simple_event(self, what, event):
2205+ """Handle a single trace event, not needing recursive processing."""
2206+ handled = True
2207+
2208+ if what == "enter":
2209+ self._add_frame(event)
2210+ elif what == "exit":
2211+ self._remove_frame(event)
2212+ else:
2213+ handled = False
2214+
2215+ return handled
2216+
2217+ def _trace(self):
2218+ """Coroutine that processes trace events it's sent."""
2219+ while True:
2220+ what, event = yield
2221+
2222+ handled = self._handle_simple_event(what, event)
2223+ if not handled:
2224+ if what == "exception":
2225+ # An exception doesn't necessarily mean the current stack
2226+ # frame is exiting. Need to check whether the next event is
2227+ # an exception in a different stack frame, implying that
2228+ # the exception is propagating up the stack.
2229+ while True:
2230+ prev_event = event
2231+ prev_name = event.qualname()
2232+ what, event = yield
2233+ if event == "exception":
2234+ if event.qualname() != prev_name:
2235+ self._remove_frame(prev_event)
2236+ else:
2237+ handled = self._handle_simple_event(what, event)
2238+ assert handled
2239+ break
2240+
2241+ else:
2242+ raise NotImplementedError
2243+
2244+ def close(self):
2245+ self._prog.commit()
2246+
2247+ def enter(self, event):
2248+ self._tracer.send(("enter", event))
2249+
2250+ def exception(self, event):
2251+ self._tracer.send(("exception", event))
2252+
2253+ def exit(self, event):
2254+ self._tracer.send(("exit", event))
2255+
2256+
2257+# @@@ config parsing shouldn't be done in __main__ (we want to get the neo4j
2258+# url from there...)
2259+@contextlib.contextmanager
2260+def trace(conn, program_name=None, filters=None):
2261+ """
2262+ Context manager that records function calls in its context block.
2263+
2264+ e.g. given this code:
2265+
2266+ with sextant.trace("http://localhost:7474"):
2267+ foo()
2268+ bar()
2269+
2270+ The calls to foo() and bar() (and their callees, at any depth) will be
2271+ recorded in the sextant database.
2272+
2273+ conn:
2274+ Instance of SextantConnection that will be used to record calls.
2275+ program_name:
2276+ String used to refer to the traced program in sextant. Defaults to
2277+ sys.argv[0].
2278+ filters:
2279+ Optional iterable of entrails filters to apply.
2280+
2281+ """
2282+ if not _entrails_available:
2283+ raise errors.MissingDependencyError(
2284+ "Entrails is required to trace execution")
2285+
2286+ if program_name is None:
2287+ program_name = sys.argv[0]
2288+
2289+ tracer = entrails.Entrails(filters=filters)
2290+ tracer.add_output(_SextantOutput(conn, program_name))
2291+
2292+ tracer.start_trace()
2293+ try:
2294+ yield
2295+ finally:
2296+ # Flush traced data.
2297+ tracer.end_trace()
2298+
2299
2300=== added file 'src/sextant/query.py'
2301--- src/sextant/query.py 1970-01-01 00:00:00 +0000
2302+++ src/sextant/query.py 2014-08-15 12:04:08 +0000
2303@@ -0,0 +1,113 @@
2304+# -----------------------------------------
2305+# Sextant
2306+# Copyright 2014, Ensoft Ltd.
2307+# Author: James Harkin, Patrick Stevens
2308+# -----------------------------------------
2309+#API for performing queries on the database
2310+
2311+#!/usr/bin/python3
2312+import argparse
2313+import requests, urllib # for different kinds of exception
2314+import logging
2315+
2316+from . import db_api
2317+from .export import ProgramConverter
2318+
2319+def query(remote_neo4j, input_query, program_name=None, argument_1=None, argument_2=None, suppress_common=False):
2320+
2321+ try:
2322+ db = db_api.SextantConnection(remote_neo4j)
2323+ except requests.exceptions.ConnectionError as err:
2324+ logging.exception("Could not connect to Neo4J server {}. Are you sure it is running?".format(remote_neo4j))
2325+ logging.exception(str(err))
2326+ return 2
2327+ #Not supported in python 2
2328+ #except (urllib.exceptions.MaxRetryError):
2329+ # logging.error("Connection was refused to {}. Are you sure the server is running?".format(remote_neo4j))
2330+ # return 2
2331+ except Exception as err:
2332+ logging.exception(str(err))
2333+ return 2
2334+
2335+ prog = None
2336+ names_list = None
2337+
2338+ if input_query == 'functions-calling':
2339+ if argument_1 == None:
2340+ print('Supply one function name to functions-calling.')
2341+ return 1
2342+ prog = db.get_all_functions_calling(program_name, argument_1)
2343+ elif input_query == 'functions-called-by':
2344+ if argument_1 == None:
2345+ print('Supply one function name to functions-called-by.')
2346+ return 1
2347+ prog = db.get_all_functions_called(program_name, argument_1)
2348+ elif input_query == 'calls-between':
2349+ if (argument_1 == None and argument_2 == None):
2350+ print('Supply two function names to calls-between.')
2351+ return 1
2352+ prog = db.get_call_paths(program_name, argument_1, argument_2)
2353+ elif input_query == 'whole-graph':
2354+ prog = db.get_whole_program(program_name)
2355+ elif input_query == 'shortest-path':
2356+ if argument_1 == None and argument_2 == None:
2357+ print('Supply two function names to shortest-path.')
2358+ return 1
2359+ prog = db.get_shortest_path_between_functions(program_name, argument_1, argument_2)
2360+ elif input_query == 'return-all-function-names':
2361+ if program_name != None:
2362+ func_names = db.get_function_names(program_name)
2363+ if func_names:
2364+ names_list = list(func_names)
2365+ else:
2366+ print('No functions were found in program %s on server %s.' % (program_name, remote_neo4j))
2367+ else:
2368+ list_of_programs = db.get_program_names()
2369+ if not list_of_programs:
2370+ print('Server %s database empty.' % (remote_neo4j))
2371+ return 0
2372+ func_list = []
2373+ for prog_name in list_of_programs:
2374+ func_list += db.get_function_names(prog_name)
2375+ if not func_list:
2376+ print('Server %s contains no functions.' % (remote_neo4j))
2377+ else:
2378+ names_list = func_list
2379+ elif input_query == 'return-all-program-names':
2380+ list_found = list(db.get_program_names())
2381+ if not list_found:
2382+ print('No programs were found on server {}.'.format(remote_neo4j))
2383+ else:
2384+ names_list = list_found
2385+ else:
2386+ print('Query unrecognised.')
2387+ return 2
2388+
2389+ if prog:
2390+ print(ProgramConverter.to_yed_graphml(prog, suppress_common))
2391+ elif names_list is not None:
2392+ print(names_list)
2393+ else:
2394+ print('Nothing was returned from the query.')
2395+
2396+
2397+def main():
2398+ argumentparser = argparse.ArgumentParser(description="Return GraphML representation or list from graph queries.")
2399+ argumentparser.add_argument('--remote-neo4j', required=True, metavar="URL", help="URL of neo4j server", type=str, nargs=1)
2400+ argumentparser.add_argument('--program-name', metavar="PROG_NAME", help="name of program as stored in the database",
2401+ type=str, nargs=1)
2402+ argumentparser.add_argument('--query', required=True, metavar="QUERY",
2403+ help="""functions-calling, functions-called-by, calls-between, whole-graph, shortest-path,
2404+ return-all-program-names or return-all-function-names; if return-all-function-names,
2405+ supply argument -program-name""", type=str, nargs=1)
2406+ argumentparser.add_argument('--funcs', metavar='FUNCS', help='functions to pass to the query', type=str, nargs='+')
2407+
2408+ parsed = argumentparser.parse_args()
2409+ names_list = None
2410+
2411+ query(remote_neo4j=parsed.remote_neo4j[0], input_query=parsed.query[0], arguments=parsed.funcs,
2412+ program_name=parsed.program_name[0])
2413+
2414+
2415+if __name__ == '__main__':
2416+ main()
2417
2418=== added file 'src/sextant/tests.py'
2419--- src/sextant/tests.py 1970-01-01 00:00:00 +0000
2420+++ src/sextant/tests.py 2014-08-15 12:04:08 +0000
2421@@ -0,0 +1,261 @@
2422+# -----------------------------------------
2423+# Sextant
2424+# Copyright 2014, Ensoft Ltd.
2425+# Author: Patrick Stevens, James Harkin
2426+# -----------------------------------------
2427+#Testing module
2428+
2429+import unittest
2430+
2431+from db_api import Function
2432+from db_api import FunctionQueryResult
2433+from db_api import SextantConnection
2434+from db_api import Validator
2435+
2436+
2437+class TestFunctionQueryResults(unittest.TestCase):
2438+ def setUp(self):
2439+ # we need to set up the remote database by using the neo4j_input_api
2440+ self.remote_url = 'http://ensoft-sandbox:7474'
2441+
2442+ self.setter_connection = SextantConnection(self.remote_url)
2443+ self.program_1_name = 'testprogram'
2444+ self.upload_program = self.setter_connection.new_program(self.program_1_name)
2445+ self.upload_program.add_function('func1')
2446+ self.upload_program.add_function('func2')
2447+ self.upload_program.add_function('func3')
2448+ self.upload_program.add_function('func4')
2449+ self.upload_program.add_function('func5')
2450+ self.upload_program.add_function('func6')
2451+ self.upload_program.add_function('func7')
2452+ self.upload_program.add_function_call('func1', 'func2')
2453+ self.upload_program.add_function_call('func1', 'func4')
2454+ self.upload_program.add_function_call('func2', 'func1')
2455+ self.upload_program.add_function_call('func2', 'func4')
2456+ self.upload_program.add_function_call('func3', 'func5')
2457+ self.upload_program.add_function_call('func4', 'func4')
2458+ self.upload_program.add_function_call('func4', 'func5')
2459+ self.upload_program.add_function_call('func5', 'func1')
2460+ self.upload_program.add_function_call('func5', 'func2')
2461+ self.upload_program.add_function_call('func5', 'func3')
2462+ self.upload_program.add_function_call('func6', 'func7')
2463+
2464+ self.upload_program.commit()
2465+
2466+ self.one_node_program_name = 'testprogram1'
2467+ self.upload_one_node_program = self.setter_connection.new_program(self.one_node_program_name)
2468+ self.upload_one_node_program.add_function('lonefunc')
2469+
2470+ self.upload_one_node_program.commit()
2471+
2472+ self.empty_program_name = 'testprogramblank'
2473+ self.upload_empty_program = self.setter_connection.new_program(self.empty_program_name)
2474+
2475+ self.upload_empty_program.commit()
2476+
2477+ self.getter_connection = SextantConnection(self.remote_url)
2478+
2479+ def tearDown(self):
2480+ self.setter_connection.delete_program(self.upload_program.program_name)
2481+ self.setter_connection.delete_program(self.upload_one_node_program.program_name)
2482+ self.setter_connection.delete_program(self.upload_empty_program.program_name)
2483+ del(self.setter_connection)
2484+
2485+ def test_17_get_call_paths(self):
2486+ reference1 = FunctionQueryResult(parent_db=None, program_name=self.program_1_name)
2487+ reference1.functions = [Function(self.program_1_name, 'func1'), Function(self.program_1_name, 'func2'),
2488+ Function(self.program_1_name, 'func3'),
2489+ Function(self.program_1_name, 'func4'), Function(self.program_1_name, 'func5')]
2490+ reference1.functions[0].functions_i_call = reference1.functions[1:4:2] # func1 calls func2, func4
2491+ reference1.functions[1].functions_i_call = reference1.functions[0:4:3] # func2 calls func1, func4
2492+ reference1.functions[2].functions_i_call = [reference1.functions[4]] # func3 calls func5
2493+ reference1.functions[3].functions_i_call = reference1.functions[3:5] # func4 calls func4, func5
2494+ reference1.functions[4].functions_i_call = reference1.functions[0:3] # func5 calls func1, func2, func3
2495+ self.assertEquals(reference1, self.getter_connection.get_call_paths(self.program_1_name, 'func1', 'func2'))
2496+ self.assertIsNone(self.getter_connection.get_call_paths('not a prog', 'func1', 'func2')) # shouldn't validation
2497+ self.assertIsNone(self.getter_connection.get_call_paths('notaprogram', 'func1', 'func2'))
2498+ self.assertIsNone(self.getter_connection.get_call_paths(self.program_1_name, 'notafunc', 'func2'))
2499+ self.assertIsNone(self.getter_connection.get_call_paths(self.program_1_name, 'func1', 'notafunc'))
2500+
2501+ def test_02_get_whole_program(self):
2502+ reference = FunctionQueryResult(parent_db=None, program_name=self.program_1_name)
2503+ reference.functions = [Function(self.program_1_name, 'func1'), Function(self.program_1_name, 'func2'),
2504+ Function(self.program_1_name, 'func3'),
2505+ Function(self.program_1_name, 'func4'), Function(self.program_1_name, 'func5'),
2506+ Function(self.program_1_name, 'func6'), Function(self.program_1_name, 'func7')]
2507+ reference.functions[0].functions_i_call = reference.functions[1:4:2] # func1 calls func2, func4
2508+ reference.functions[1].functions_i_call = reference.functions[0:4:3] # func2 calls func1, func4
2509+ reference.functions[2].functions_i_call = [reference.functions[4]] # func3 calls func5
2510+ reference.functions[3].functions_i_call = reference.functions[3:5] # func4 calls func4, func5
2511+ reference.functions[4].functions_i_call = reference.functions[0:3] # func5 calls func1, func2, func3
2512+ reference.functions[5].functions_i_call = [reference.functions[6]] # func6 calls func7
2513+
2514+
2515+ self.assertEqual(reference, self.getter_connection.get_whole_program(self.program_1_name))
2516+ self.assertIsNone(self.getter_connection.get_whole_program('nottherightprogramname'))
2517+
2518+ def test_03_get_whole_one_node_program(self):
2519+ reference = FunctionQueryResult(parent_db=None, program_name=self.one_node_program_name)
2520+ reference.functions = [Function(self.one_node_program_name, 'lonefunc')]
2521+
2522+ self.assertEqual(reference, self.getter_connection.get_whole_program(self.one_node_program_name))
2523+
2524+ def test_04_get_whole_empty_program(self):
2525+ reference = FunctionQueryResult(parent_db=None, program_name=self.empty_program_name)
2526+ reference.functions = []
2527+
2528+ self.assertEqual(reference, self.getter_connection.get_whole_program(self.empty_program_name))
2529+
2530+ def test_05_get_function_names(self):
2531+ reference = {'func1', 'func2', 'func3', 'func4', 'func5', 'func6', 'func7'}
2532+ self.assertEqual(reference, self.getter_connection.get_function_names(self.program_1_name))
2533+
2534+ def test_06_get_function_names_one_node_program(self):
2535+ reference = {'lonefunc'}
2536+ self.assertEqual(reference, self.getter_connection.get_function_names(self.one_node_program_name))
2537+
2538+ def test_07_get_function_names_empty_program(self):
2539+ reference = set()
2540+ self.assertEqual(reference, self.getter_connection.get_function_names(self.empty_program_name))
2541+
2542+ def test_09_validation_is_used(self):
2543+ self.assertFalse(self.getter_connection.get_function_names('not alphanumeric'))
2544+ self.assertFalse(self.getter_connection.get_whole_program('not alphanumeric'))
2545+ self.assertFalse(self.getter_connection.check_program_exists('not alphanumeric'))
2546+ self.assertFalse(self.getter_connection.check_function_exists('not alphanumeric', 'alpha'))
2547+ self.assertFalse(self.getter_connection.check_function_exists('alpha', 'not alpha'))
2548+ self.assertFalse(self.getter_connection.get_all_functions_called('alphaprogram', 'not alpha function'))
2549+ self.assertFalse(self.getter_connection.get_all_functions_called('not alpha program', 'alphafunction'))
2550+ self.assertFalse(self.getter_connection.get_all_functions_calling('not alpha program', 'alphafunction'))
2551+ self.assertFalse(self.getter_connection.get_all_functions_calling('alphaprogram', 'not alpha function'))
2552+ self.assertFalse(self.getter_connection.get_call_paths('not alpha program','alphafunc1', 'alphafunc2'))
2553+ self.assertFalse(self.getter_connection.get_call_paths('alphaprogram','not alpha func 1', 'alphafunc2'))
2554+ self.assertFalse(self.getter_connection.get_call_paths('alphaprogram','alphafunc1', 'not alpha func 2'))
2555+
2556+ def test_08_get_program_names(self):
2557+ reference = {self.program_1_name, self.one_node_program_name, self.empty_program_name}
2558+ self.assertEqual(reference, self.getter_connection.get_program_names())
2559+
2560+
2561+ def test_11_get_all_functions_called(self):
2562+ reference1 = FunctionQueryResult(parent_db=None, program_name=self.program_1_name) # this will be the 1,2,3,4,5 component
2563+ reference2 = FunctionQueryResult(parent_db=None, program_name=self.program_1_name) # this will be the 6,7 component
2564+ reference3 = FunctionQueryResult(parent_db=None, program_name=self.program_1_name) # this will be the 7 component
2565+ reference1.functions = [Function(self.program_1_name, 'func1'), Function(self.program_1_name, 'func2'),
2566+ Function(self.program_1_name, 'func3'),
2567+ Function(self.program_1_name, 'func4'), Function(self.program_1_name, 'func5')]
2568+ reference2.functions = [Function(self.program_1_name, 'func6'), Function(self.program_1_name, 'func7')]
2569+ reference3.functions = []
2570+
2571+ reference1.functions[0].functions_i_call = reference1.functions[1:4:2] # func1 calls func2, func4
2572+ reference1.functions[1].functions_i_call = reference1.functions[0:4:3] # func2 calls func1, func4
2573+ reference1.functions[2].functions_i_call = [reference1.functions[4]] # func3 calls func5
2574+ reference1.functions[3].functions_i_call = reference1.functions[3:5] # func4 calls func4, func5
2575+ reference1.functions[4].functions_i_call = reference1.functions[0:3] # func5 calls func1, func2, func3
2576+
2577+ reference2.functions[0].functions_i_call = [reference2.functions[1]]
2578+
2579+ self.assertEquals(reference1, self.getter_connection.get_all_functions_called(self.program_1_name, 'func1'))
2580+ self.assertEquals(reference1, self.getter_connection.get_all_functions_called(self.program_1_name, 'func2'))
2581+ self.assertEquals(reference1, self.getter_connection.get_all_functions_called(self.program_1_name, 'func3'))
2582+ self.assertEquals(reference1, self.getter_connection.get_all_functions_called(self.program_1_name, 'func4'))
2583+ self.assertEquals(reference1, self.getter_connection.get_all_functions_called(self.program_1_name, 'func5'))
2584+
2585+ self.assertEquals(reference2, self.getter_connection.get_all_functions_called(self.program_1_name, 'func6'))
2586+
2587+ self.assertEquals(reference3, self.getter_connection.get_all_functions_called(self.program_1_name, 'func7'))
2588+
2589+ self.assertIsNone(self.getter_connection.get_all_functions_called(self.program_1_name, 'nottherightfunction'))
2590+ self.assertIsNone(self.getter_connection.get_all_functions_called('nottherightprogram', 'func2'))
2591+
2592+ def test_12_get_all_functions_called_1(self):
2593+ reference1 = FunctionQueryResult(parent_db=None, program_name=self.one_node_program_name)
2594+ reference1.functions = []
2595+
2596+ d=self.getter_connection.get_all_functions_called(self.one_node_program_name, 'lonefunc')
2597+ self.assertEquals(reference1, self.getter_connection.get_all_functions_called(self.one_node_program_name,
2598+ 'lonefunc'))
2599+ self.assertIsNone(self.getter_connection.get_all_functions_called(self.one_node_program_name,
2600+ 'not the right function'))
2601+ self.assertIsNone(self.getter_connection.get_all_functions_called('not the right program', 'lonefunc'))
2602+
2603+ def test_13_get_all_functions_called_blank(self):
2604+ reference1 = FunctionQueryResult(parent_db=None, program_name=self.empty_program_name)
2605+ reference1.functions = []
2606+
2607+ self.assertIsNone(self.getter_connection.get_all_functions_called(self.empty_program_name,
2608+ 'not the right function'))
2609+
2610+ def test_14_get_all_functions_calling(self):
2611+ reference1 = FunctionQueryResult(parent_db=None, program_name=self.program_1_name) # this will be the 1,2,3,4,5 component
2612+ reference2 = FunctionQueryResult(parent_db=None, program_name=self.program_1_name) # this will be the 6,7 component
2613+ reference3 = FunctionQueryResult(parent_db=None, program_name=self.program_1_name) # this will be the 7 component
2614+ reference1.functions = [Function(self.program_1_name, 'func1'), Function(self.program_1_name, 'func2'),
2615+ Function(self.program_1_name, 'func3'),
2616+ Function(self.program_1_name, 'func4'), Function(self.program_1_name, 'func5')]
2617+
2618+ reference1.functions[0].functions_i_call = reference1.functions[1:4:2] # func1 calls func2, func4
2619+ reference1.functions[1].functions_i_call = reference1.functions[0:4:3] # func2 calls func1, func4
2620+ reference1.functions[2].functions_i_call = [reference1.functions[4]] # func3 calls func5
2621+ reference1.functions[3].functions_i_call = reference1.functions[3:5] # func4 calls func4, func5
2622+ reference1.functions[4].functions_i_call = reference1.functions[0:3] # func5 calls func1, func2, func3
2623+
2624+ reference2.functions = [Function(self.program_1_name, 'func6'), Function(self.program_1_name, 'func7')]
2625+
2626+ reference2.functions[0].functions_i_call = [reference2.functions[1]]
2627+
2628+ reference3.functions = [Function(self.program_1_name, 'func6')]
2629+
2630+ reference3.functions = []
2631+
2632+ self.assertEquals(reference1, self.getter_connection.get_all_functions_calling(self.program_1_name, 'func1'))
2633+ self.assertEquals(reference1, self.getter_connection.get_all_functions_calling(self.program_1_name, 'func2'))
2634+ self.assertEquals(reference1, self.getter_connection.get_all_functions_calling(self.program_1_name, 'func3'))
2635+ self.assertEquals(reference1, self.getter_connection.get_all_functions_calling(self.program_1_name, 'func4'))
2636+ self.assertEquals(reference1, self.getter_connection.get_all_functions_calling(self.program_1_name, 'func5'))
2637+
2638+ self.assertEquals(reference2, self.getter_connection.get_all_functions_calling(self.program_1_name,'func7'))
2639+
2640+ self.assertEquals(reference3, self.getter_connection.get_all_functions_calling(self.program_1_name, 'func6'))
2641+
2642+ self.assertIsNone(self.getter_connection.get_all_functions_calling(self.program_1_name, 'nottherightfunction'))
2643+ self.assertIsNone(self.getter_connection.get_all_functions_calling('nottherightprogram', 'func2'))
2644+
2645+ def test_15_get_all_functions_calling_one_node_prog(self):
2646+ reference1 = FunctionQueryResult(parent_db=None, program_name=self.one_node_program_name)
2647+ reference1.functions = []
2648+ self.assertEquals(reference1, self.getter_connection.get_all_functions_calling(self.one_node_program_name,
2649+ 'lonefunc'))
2650+ self.assertIsNone(self.getter_connection.get_all_functions_calling(self.one_node_program_name,
2651+ 'not the right function'))
2652+ self.assertIsNone(self.getter_connection.get_all_functions_calling('not the right program', 'lonefunc'))
2653+
2654+ def test_16_get_all_functions_calling_blank_prog(self):
2655+ reference1 = FunctionQueryResult(parent_db=None, program_name=self.empty_program_name)
2656+ reference1.functions=[]
2657+
2658+ self.assertIsNone(self.getter_connection.get_all_functions_called(self.empty_program_name,
2659+ 'not the right function'))
2660+
2661+
2662+
2663+ def test_18_get_call_paths_between_two_functions_one_node_prog(self):
2664+ reference = FunctionQueryResult(parent_db=None, program_name=self.one_node_program_name)
2665+ reference.functions = [] # that is, reference is the empty program with name self.one_node_program_name
2666+
2667+ self.assertEquals(self.getter_connection.get_call_paths(self.one_node_program_name, 'lonefunc', 'lonefunc'),
2668+ reference)
2669+ self.assertIsNone(self.getter_connection.get_call_paths(self.one_node_program_name, 'lonefunc', 'notafunc'))
2670+ self.assertIsNone(self.getter_connection.get_call_paths(self.one_node_program_name, 'notafunc', 'notafunc'))
2671+
2672+ def test_10_validator(self):
2673+ self.assertFalse(Validator.validate(''))
2674+ self.assertTrue(Validator.validate('thisworks'))
2675+ self.assertTrue(Validator.validate('th1sw0rks'))
2676+ self.assertTrue(Validator.validate('12345'))
2677+ self.assertFalse(Validator.validate('this does not work'))
2678+ self.assertTrue(Validator.validate('this_does_work'))
2679+ self.assertFalse(Validator.validate("'")) # string consisting of a single quote mark
2680+
2681+if __name__ == '__main__':
2682+ unittest.main()
2683\ No newline at end of file
2684
2685=== added file 'src/sextant/update_db.py'
2686--- src/sextant/update_db.py 1970-01-01 00:00:00 +0000
2687+++ src/sextant/update_db.py 2014-08-15 12:04:08 +0000
2688@@ -0,0 +1,78 @@
2689+# -----------------------------------------
2690+# Sextant
2691+# Copyright 2014, Ensoft Ltd.
2692+# Author: Patrick Stevens, using work from Patrick Stevens and James Harkin
2693+# -----------------------------------------
2694+# Given a program file to upload, or a program name to delete from the server, does the right thing.
2695+
2696+__all__ = ("upload_program", "delete_program")
2697+
2698+from .db_api import SextantConnection, Validator
2699+from .objdump_parser import get_parsed_objects
2700+
2701+import logging
2702+import requests
2703+
2704+
2705+def upload_program(file_path, db_url, alternative_name=None, not_object_file=False):
2706+ """
2707+ Uploads a program to the remote database.
2708+ :param file_path: the path to the local file we wish to upload
2709+ :param db_url: the URL of the database (eg. http://localhost:7474)
2710+ :param alternative_name: a name to give the program to override the default
2711+ :param object_file: bool(the file is an objdump text output file, rather than a compiled binary)
2712+ :return: 1 if the program already exists in database, 2 if there was a connection error
2713+ """
2714+ try:
2715+ connection = SextantConnection(db_url)
2716+ except requests.exceptions.ConnectionError as err:
2717+ logging.exception("Could not connect to Neo4J server {}. Are you sure it is running?".format(db_url))
2718+ return 2
2719+ #except urllib.exceptions.MaxRetryError:
2720+ # logging.error("Connection was refused to {}. Are you sure the server is running?".format(db_url))
2721+ # return 2
2722+
2723+ program_names = connection.get_program_names()
2724+ if alternative_name is None:
2725+ if Validator.sanitise(file_path) in program_names:
2726+ logging.error("There is already a program under this name; please delete the previous one with the same name "
2727+ "and retry, or rename the input file.")
2728+ return 1
2729+ else:
2730+ if Validator.sanitise(alternative_name) in program_names:
2731+ logging.error("There is already a program under this name; please delete the previous one with the same name "
2732+ "and retry, or rename the input file.")
2733+ return 1
2734+
2735+ parsed_objects = get_parsed_objects(filepath=file_path, sections_to_view=['.text'],
2736+ not_object_file=not_object_file, ignore_function_pointers=False)
2737+
2738+ logging.warning('Objdump has parsed!')
2739+
2740+ if alternative_name is None:
2741+ program_representation = connection.new_program(Validator.sanitise(file_path))
2742+ else:
2743+ program_representation = connection.new_program(Validator.sanitise(alternative_name))
2744+
2745+
2746+
2747+ for obj in parsed_objects:
2748+ for called in obj.what_do_i_call:
2749+ if not program_representation.add_function_call(obj.name, called[-1]): # called is a tuple (address, name)
2750+ logging.error('Validation error: {} calling {}'.format(obj.name, called[-1]))
2751+
2752+ logging.warning('Sending {} named objects to server {}...'.format(len(parsed_objects), db_url))
2753+ program_representation.commit()
2754+ logging.info('Sending complete! Exiting.')
2755+
2756+
2757+def delete_program(program_name, db_url):
2758+ """
2759+ Deletes a program with the specified name from the database.
2760+ :param program_name: the name of the program to delete
2761+ :param db_url: the URL of the database (eg. http://localhost:7474)
2762+ :return: bool(success)
2763+ """
2764+ connection = SextantConnection(db_url)
2765+ connection.delete_program(program_name)
2766+ print('Deleted {} successfully.'.format(program_name))
2767
2768=== added directory 'src/sextant/web'
2769=== added file 'src/sextant/web/__init__.py'
2770--- src/sextant/web/__init__.py 1970-01-01 00:00:00 +0000
2771+++ src/sextant/web/__init__.py 2014-08-15 12:04:08 +0000
2772@@ -0,0 +1,8 @@
2773+#----------------------------------------------------------------------------
2774+# __init__.py -- Web package root
2775+#
2776+# July 2014, Phil Connell
2777+#
2778+# (c) Ensoft Ltd, 2014
2779+#----------------------------------------------------------------------------
2780+
2781
2782=== added file 'src/sextant/web/server.py'
2783--- src/sextant/web/server.py 1970-01-01 00:00:00 +0000
2784+++ src/sextant/web/server.py 2014-08-15 12:04:08 +0000
2785@@ -0,0 +1,323 @@
2786+#!/usr/bin/python2
2787+# -----------------------------------------
2788+# Sextant
2789+# Copyright 2014, Ensoft Ltd.
2790+# Author: Patrick Stevens, James Harkin
2791+# -----------------------------------------
2792+# Note: this must be run in Python 2.
2793+
2794+from twisted.web.server import Site, NOT_DONE_YET
2795+from twisted.web.resource import Resource
2796+from twisted.web.static import File
2797+from twisted.internet import reactor
2798+from twisted.internet.threads import deferToThread
2799+from twisted.internet import defer
2800+
2801+import logging
2802+
2803+import os
2804+import sys # hack to get the Sextant module imported
2805+sys.path.append(os.path.realpath('../..'))
2806+
2807+import json
2808+import requests
2809+
2810+import sextant.db_api as db_api
2811+import sextant.export as export
2812+import tempfile
2813+import subprocess
2814+
2815+from cgi import escape # deprecated in Python 3 in favour of html.escape, but we're stuck on Python 2
2816+
2817+database_url = None # the URL to access the database instance
2818+
2819+class Echoer(Resource):
2820+ # designed to take one name argument
2821+
2822+ def render_GET(self, request):
2823+ if "name" not in request.args:
2824+ return '<html><body>Greetings, unnamed stranger.</body></html>'
2825+
2826+ arg = escape(request.args["name"][0])
2827+ return '<html><body>Hello %s!</body></html>' % arg
2828+
2829+
2830+class SVGRenderer(Resource):
2831+
2832+ def error_creating_neo4j_connection(self, failure):
2833+ self.write("Error creating Neo4J connection: %s\n") % failure.getErrorMessage()
2834+
2835+ @staticmethod
2836+ def create_neo4j_connection():
2837+ return db_api.SextantConnection(database_url)
2838+
2839+ @staticmethod
2840+ def check_program_exists(connection, name):
2841+ return connection.check_program_exists(name)
2842+
2843+ @staticmethod
2844+ def get_whole_program(connection, name):
2845+ return connection.get_whole_program(name)
2846+
2847+ @staticmethod
2848+ def get_functions_calling(connection, progname, funcname):
2849+ return connection.get_all_functions_calling(progname, funcname)
2850+
2851+ @staticmethod
2852+ def get_plot(program, suppress_common_functions=False):
2853+ graph_dot = export.ProgramConverter.to_dot(program, suppress_common_functions)
2854+
2855+ file_written_to = tempfile.NamedTemporaryFile(delete=False)
2856+ file_out = tempfile.NamedTemporaryFile(delete=False)
2857+ file_written_to.write(graph_dot)
2858+ file_written_to.close()
2859+ subprocess.call(['dot', '-Tsvg', '-Kdot', '-o', file_out.name, file_written_to.name])
2860+
2861+ output = file_out.read().encode()
2862+ file_out.close()
2863+ return output
2864+
2865+ @defer.inlineCallbacks
2866+ def _render_plot(self, request):
2867+ if "program_name" not in request.args:
2868+ request.setResponseCode(400)
2869+ request.write("Supply 'program_name' parameter.")
2870+ request.finish()
2871+ defer.returnValue(None)
2872+
2873+ logging.info('enter')
2874+ name = request.args["program_name"][0]
2875+
2876+ try:
2877+ suppress_common = request.args["suppress_common"][0]
2878+ except KeyError:
2879+ suppress_common = False
2880+
2881+ if suppress_common == 'null' or suppress_common == 'true':
2882+ suppress_common = True
2883+ else:
2884+ suppress_common = False
2885+
2886+ try:
2887+ neo4jconnection = yield deferToThread(self.create_neo4j_connection)
2888+ except requests.exceptions.ConnectionError:
2889+ request.setResponseCode(502) # Bad Gateway
2890+ request.write("Could not reach Neo4j server at {}".format(database_url))
2891+ request.finish()
2892+ defer.returnValue(None)
2893+ neo4jconnection = None # to silence the "referenced before assignment" warnings later
2894+
2895+ logging.info('created')
2896+ exists = yield deferToThread(self.check_program_exists, neo4jconnection, name)
2897+ if not exists:
2898+ request.setResponseCode(404)
2899+ logging.info('returning nonexistent')
2900+ request.write("Name %s not found." % (escape(name)))
2901+ request.finish()
2902+ defer.returnValue(None)
2903+
2904+ logging.info('done created')
2905+ allowed_queries = ("whole_program", "functions_calling", "functions_called_by", "call_paths", "shortest_path")
2906+
2907+ if "query" not in request.args:
2908+ query = "whole_program"
2909+ else:
2910+ query = request.args["query"][0]
2911+
2912+ if query not in allowed_queries:
2913+ # raise 400 Bad Request error
2914+ request.setResponseCode(400)
2915+ request.write("Supply 'query' parameter, default is whole_program, allowed %s." % str(allowed_queries))
2916+ request.finish()
2917+ defer.returnValue(None)
2918+
2919+ if query == 'whole_program':
2920+ program = yield deferToThread(self.get_whole_program, neo4jconnection, name)
2921+ elif query == 'functions_calling':
2922+ if 'func1' not in request.args:
2923+ # raise 400 Bad Request error
2924+ request.setResponseCode(400)
2925+ request.write("Supply 'func1' parameter to functions_calling.")
2926+ request.finish()
2927+ defer.returnValue(None)
2928+ func1 = request.args['func1'][0]
2929+ program = yield deferToThread(self.get_functions_calling, neo4jconnection, name, func1)
2930+ elif query == 'functions_called_by':
2931+ if 'func1' not in request.args:
2932+ # raise 400 Bad Request error
2933+ request.setResponseCode(400)
2934+ request.write("Supply 'func1' parameter to functions_called_by.")
2935+ request.finish()
2936+ defer.returnValue(None)
2937+ func1 = request.args['func1'][0]
2938+ program = yield deferToThread(neo4jconnection.get_all_functions_called, name, func1)
2939+ elif query == 'call_paths':
2940+ if 'func1' not in request.args:
2941+ # raise 400 Bad Request error
2942+ request.setResponseCode(400)
2943+ request.write("Supply 'func1' parameter to call_paths.")
2944+ request.finish()
2945+ defer.returnValue(None)
2946+ if 'func2' not in request.args:
2947+ # raise 400 Bad Request error
2948+ request.setResponseCode(400)
2949+ request.write("Supply 'func2' parameter to call_paths.")
2950+ request.finish()
2951+ defer.returnValue(None)
2952+
2953+ func1 = request.args['func1'][0]
2954+ func2 = request.args['func2'][0]
2955+ program = yield deferToThread(neo4jconnection.get_call_paths, name, func1, func2)
2956+ elif query == 'shortest_path':
2957+ if 'func1' not in request.args:
2958+ # raise 400 Bad Request error
2959+ request.setResponseCode(400)
2960+ request.write("Supply 'func1' parameter to shortest_path.")
2961+ request.finish()
2962+ defer.returnValue(None)
2963+ if 'func2' not in request.args:
2964+ # raise 400 Bad Request error
2965+ request.setResponseCode(400)
2966+ request.write("Supply 'func2' parameter to shortest_path.")
2967+ request.finish()
2968+ defer.returnValue(None)
2969+
2970+ func1 = request.args['func1'][0]
2971+ func2 = request.args['func2'][0]
2972+ program = yield deferToThread(neo4jconnection.get_shortest_path_between_functions, name, func1, func2)
2973+
2974+ else: # unrecognised query, so we need to raise a Bad Request response
2975+ request.setResponseCode(400)
2976+ request.write("Query %s not recognised." % escape(query))
2977+ request.finish()
2978+ defer.returnValue(None)
2979+ program = None # silences the "referenced before assignment" warnings
2980+
2981+ if program is None:
2982+ request.setResponseCode(404)
2983+ request.write("At least one of the input functions was not found in program %s." % (escape(name)))
2984+ request.finish()
2985+ defer.returnValue(None)
2986+
2987+ logging.info('getting plot')
2988+ logging.info(program)
2989+ if not program.functions: # we got an empty program back: the program is in the Sextant but has no functions
2990+ request.setResponseCode(204)
2991+ request.finish()
2992+ defer.returnValue(None)
2993+
2994+ output = yield deferToThread(self.get_plot, program, suppress_common)
2995+ request.setHeader("content-type", "image/svg+xml")
2996+
2997+ logging.info('SVG: return')
2998+ request.write(output)
2999+ request.finish()
3000+
3001+ def render_GET(self, request):
3002+ self._render_plot(request)
3003+ return NOT_DONE_YET
3004+
3005+
3006+class GraphProperties(Resource):
3007+
3008+ @staticmethod
3009+ def _get_connection():
3010+ return db_api.SextantConnection(database_url)
3011+
3012+ @staticmethod
3013+ def _get_program_names(connection):
3014+ return connection.get_program_names()
3015+
3016+ @staticmethod
3017+ def _get_function_names(connection, program_name):
3018+ return connection.get_function_names(program_name)
3019+
3020+ @defer.inlineCallbacks
3021+ def _render_GET(self, request):
3022+ if "query" not in request.args:
3023+ request.setResponseCode(400)
3024+ request.setHeader("content-type", "text/plain")
3025+ request.write("Supply 'query' parameter of 'programs' or 'functions'.")
3026+ request.finish()
3027+ defer.returnValue(None)
3028+
3029+ query = request.args['query'][0]
3030+
3031+ logging.info('Properties: about to get_connection')
3032+
3033+ try:
3034+ neo4j_connection = yield deferToThread(self._get_connection)
3035+ except Exception:
3036+ request.setResponseCode(502) # Bad Gateway
3037+ request.write("Could not reach Neo4j server at {}.".format(database_url))
3038+ request.finish()
3039+ defer.returnValue(None)
3040+ neo4j_connection = None # just to silence the "referenced before assignment" warnings
3041+
3042+ logging.info('got connection')
3043+
3044+ if query == 'programs':
3045+ request.setHeader("content-type", "application/json")
3046+ prognames = yield deferToThread(self._get_program_names, neo4j_connection)
3047+ request.write(json.dumps(list(prognames)))
3048+ request.finish()
3049+ defer.returnValue(None)
3050+
3051+ elif query == 'functions':
3052+ if "program_name" not in request.args:
3053+ request.setResponseCode(400)
3054+ request.setHeader("content-type", "text/plain")
3055+ request.write("Supply 'program_name' parameter to ?query=functions.")
3056+ request.finish()
3057+ defer.returnValue(None)
3058+ program_name = request.args['program_name'][0]
3059+
3060+ funcnames = yield deferToThread(self._get_function_names, neo4j_connection, program_name)
3061+ if funcnames is None:
3062+ request.setResponseCode(404)
3063+ request.setHeader("content-type", "text/plain")
3064+ request.write("No program with name %s was found in the Sextant." % escape(program_name))
3065+ request.finish()
3066+ defer.returnValue(None)
3067+
3068+ request.setHeader("content-type", "application/json")
3069+ request.write(json.dumps(list(funcnames)))
3070+ request.finish()
3071+ defer.returnValue(None)
3072+
3073+ else:
3074+ request.setResponseCode(400)
3075+ request.setHeader("content-type", "text/plain")
3076+ request.write("'Query' parameter should be 'programs' or 'functions'.")
3077+ request.finish()
3078+ defer.returnValue(None)
3079+
3080+ def render_GET(self, request):
3081+ self._render_GET(request)
3082+ return NOT_DONE_YET
3083+
3084+
3085+def serve_site(input_database_url='http://localhost:7474', port=2905):
3086+
3087+ global database_url
3088+ database_url = input_database_url
3089+ # serve static directory at root
3090+ root = File(os.path.join(
3091+ os.path.dirname(os.path.abspath(__file__)),
3092+ "..", "..", "..", "resources", "web"))
3093+
3094+ # serve a dynamic Echoer webpage at /echoer.html
3095+ root.putChild("echoer.html", Echoer())
3096+
3097+ # serve a dynamic webpage at /Sextant_properties to return graph properties
3098+ root.putChild("database_properties", GraphProperties())
3099+
3100+ # serve a generated SVG at /output_graph.svg
3101+ root.putChild('output_graph.svg', SVGRenderer())
3102+
3103+ factory = Site(root)
3104+ reactor.listenTCP(port, factory)
3105+ reactor.run()
3106+
3107+if __name__ == '__main__':
3108+ serve_site()

Subscribers

People subscribed via source and target branches