Merge lp:~ben-hutchings/ensoft-sextant/wierd-names-clean into lp:ensoft-sextant
- wierd-names-clean
- Merge into whiteline
Proposed by
Ben Hutchings
Status: | Superseded |
---|---|
Proposed branch: | lp:~ben-hutchings/ensoft-sextant/wierd-names-clean |
Merge into: | lp:ensoft-sextant |
Diff against target: |
1374 lines (+494/-427) 11 files modified
resources/sextant/web/interface.html (+10/-3) resources/sextant/web/queryjavascript.js (+7/-2) src/sextant/db_api.py (+248/-99) src/sextant/export.py (+9/-5) src/sextant/objdump_parser.py (+68/-18) src/sextant/test_db.py (+97/-0) src/sextant/test_db_api.py (+0/-275) src/sextant/test_parser.py (+21/-17) src/sextant/test_resources/parser_test.dump (+25/-0) src/sextant/update_db.py (+3/-3) src/sextant/web/server.py (+6/-5) |
To merge this branch: | bzr merge lp:~ben-hutchings/ensoft-sextant/wierd-names-clean |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Robert | Pending | ||
Review via email: mp+242751@code.launchpad.net |
This proposal has been superseded by a proposal from 2014-11-25.
Commit message
Merge functions which end in '.' or '..<digit>'.
Description of the change
Merge functions which end in '.' or '..<digit>'.
To post a comment you must log in.
- 64. By Ben Hutchings
-
merge from rel-merge markups
- 65. By Ben Hutchings
-
extended tests, replaced test_all.sh with test_all.py which also generates code coverage reports for the tested modeuls
- 66. By Ben Hutchings
-
whitespace fix
- 67. By Ben Hutchings
-
fixed bug with get_all_
functions_ calling - 68. By Ben Hutchings
-
very minor change to the default objdump_parser print output format to include more information added since this was originally written
- 69. By Ben Hutchings
-
doc comment fix
Unmerged revisions
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'resources/sextant/web/interface.html' |
2 | --- resources/sextant/web/interface.html 2014-11-13 10:47:34 +0000 |
3 | +++ resources/sextant/web/interface.html 2014-11-25 10:17:59 +0000 |
4 | @@ -27,8 +27,8 @@ |
5 | All functions calling specific function</option> |
6 | <option value="functions_called_by"> |
7 | All functions called by a specific function</option> |
8 | - <!--option value="all_call_paths"> REMOVED AS THIS IS SLOW FOR IOS |
9 | - All function call paths between two functions</option--> |
10 | + <option value="all_call_paths"> |
11 | + All function call paths between two functions</option> |
12 | <option value="shortest_call_path"> |
13 | Shortest path between two functions</option> |
14 | <option value="function_names"> |
15 | @@ -39,7 +39,14 @@ |
16 | <input type="checkbox" id="suppress_common" value="True"></input> |
17 | Suppress common functions? |
18 | </label> |
19 | - |
20 | + <label> |
21 | + <input type="checkbox" id="limit_internal" value="True"></input> |
22 | + Limit to internal calls? |
23 | + </label> |
24 | + <label> |
25 | + <input type="number" id="max_depth" value="1" min="0" max="10"></input> |
26 | + Maximum call depth. |
27 | + </label> |
28 | <button class="button" style="float:right; margin: 1px 20px -1px 0;" onclick="execute_query()">Run Query</button> |
29 | </div> |
30 | <div id="toolbar-row2" style="margin-left: 234px;"> |
31 | |
32 | === modified file 'resources/sextant/web/queryjavascript.js' |
33 | --- resources/sextant/web/queryjavascript.js 2014-11-21 12:50:51 +0000 |
34 | +++ resources/sextant/web/queryjavascript.js 2014-11-25 10:17:59 +0000 |
35 | @@ -157,7 +157,7 @@ |
36 | } else { |
37 | //If not function names we will want a graph as an output; |
38 | //url returns svg file of graph. |
39 | - // We use a random number argument to prevent caching. |
40 | + // We use a random number argument to prevent caching. |
41 | var string = "/output_graph.svg?stop_cache=" + String(Math.random()) + "&program_name=" + |
42 | document.getElementById("program_name").value + |
43 | "&query=" + query_id + "&func1="; |
44 | @@ -165,8 +165,13 @@ |
45 | "&func2=" + document.getElementById("function_2").value; |
46 | string = string + "&suppress_common=" + |
47 | document.getElementById('suppress_common').checked.toString(); |
48 | + string = string + "&limit_internal=" + |
49 | + document.getElementById('limit_internal').checked.toString(); |
50 | + string = string + "&max_depth=" + |
51 | + document.getElementById('max_depth').value.toString(); |
52 | + } |
53 | + |
54 | |
55 | - } |
56 | var xmlhttp = new XMLHttpRequest(); |
57 | xmlhttp.open("GET", string, true); |
58 | xmlhttp.send(); |
59 | |
60 | === modified file 'src/sextant/db_api.py' |
61 | --- src/sextant/db_api.py 2014-11-19 10:40:24 +0000 |
62 | +++ src/sextant/db_api.py 2014-11-25 10:17:59 +0000 |
63 | @@ -162,7 +162,7 @@ |
64 | headers=['name', 'type', 'file'], |
65 | max_rows=5000) |
66 | self.call_writer = CSVWriter(tmp_path.format('calls'), |
67 | - headers=['caller', 'callee'], |
68 | + headers=['caller', 'callee', 'is_internal'], |
69 | max_rows=5000) |
70 | |
71 | # Define the queries we use to upload the functions and calls. |
72 | @@ -173,14 +173,26 @@ |
73 | ' CREATE (n)-[:subject]->(m:func {{name: line.name,' |
74 | ' id: lineid, type: line.type, file: line.file}})') |
75 | |
76 | - self.add_call_query = (' USING PERIODIC COMMIT 250' |
77 | - ' LOAD CSV WITH HEADERS FROM "file:{}" AS line' |
78 | - ' MATCH (p:program {{name: "{}"}})' |
79 | - ' MATCH (p)-[:subject]->(n:func {{name: line.caller}})' |
80 | - ' USING INDEX n:func(name)' |
81 | - ' MATCH (p)-[:subject]->(m:func {{name: line.callee}})' |
82 | - ' USING INDEX m:func(name)' |
83 | - ' CREATE (n)-[r:calls]->(m)') |
84 | + self.add_internal_call_query = (' USING PERIODIC COMMIT 250' |
85 | + ' LOAD CSV WITH HEADERS FROM "file:{}" AS line' |
86 | + ' WITH line WHERE line.is_internal = "True"' |
87 | + ' MATCH (p:program {{name: "{}"}})' |
88 | + ' MATCH (p)-[:subject]->(n:func {{name: line.caller}})' |
89 | + ' USING INDEX n:func(name)' |
90 | + ' MATCH (p)-[:subject]->(m:func {{name: line.callee}})' |
91 | + ' USING INDEX m:func(name)' |
92 | + ' CREATE (n)-[r:internal]->(m)') |
93 | + |
94 | + self.add_external_call_query = (' USING PERIODIC COMMIT 250' |
95 | + ' LOAD CSV WITH HEADERS FROM "file:{}" AS line' |
96 | + ' WITH line WHERE line.is_internal <> "True"' |
97 | + ' MATCH (p:program {{name: "{}"}})' |
98 | + ' MATCH (p)-[:subject]->(n:func {{name: line.caller}})' |
99 | + ' USING INDEX n:func(name)' |
100 | + ' MATCH (p)-[:subject]->(m:func {{name: line.callee}})' |
101 | + ' USING INDEX m:func(name)' |
102 | + ' CREATE (n)-[r:external]->(m)') |
103 | + |
104 | |
105 | self.add_program_query = ('CREATE (p:program {{name: "{}", uploader: "{}", ' |
106 | ' uploader_id: "{}", date: "{}",' |
107 | @@ -221,7 +233,7 @@ |
108 | """ |
109 | self.func_writer.write(name, typ, source) |
110 | |
111 | - def add_call(self, caller, callee): |
112 | + def add_call(self, caller, callee, is_internal=False): |
113 | """ |
114 | Add a function call. |
115 | |
116 | @@ -230,8 +242,11 @@ |
117 | The name of the function making the call. |
118 | callee: |
119 | The name of the function called. |
120 | + is_internal: |
121 | + True if the caller's source file is the same as callee's, |
122 | + unless either one is 'unknown'. |
123 | """ |
124 | - self.call_writer.write(caller, callee) |
125 | + self.call_writer.write(caller, callee, is_internal) |
126 | |
127 | |
128 | def _copy_local_to_remote_tmp_dir(self): |
129 | @@ -257,7 +272,6 @@ |
130 | remote_paths: |
131 | A list of the paths of the remote fils. |
132 | """ |
133 | - |
134 | def try_rmdir(path): |
135 | # Helper function to try and remove a directory, silently |
136 | # fail if it contains files, otherwise raise the exception. |
137 | @@ -270,6 +284,7 @@ |
138 | else: |
139 | raise e |
140 | |
141 | + |
142 | print('Cleaning temporary files...', end='') |
143 | file_paths = list(itertools.chain(self.func_writer.file_iter(), |
144 | self.call_writer.file_iter())) |
145 | @@ -279,7 +294,7 @@ |
146 | |
147 | try_rmdir(self._tmp_dir) |
148 | try_rmdir(TMP_DIR) |
149 | - |
150 | + |
151 | self._ssh.remove_from_tmp_dir(remote_paths) |
152 | |
153 | print('done.') |
154 | @@ -336,9 +351,12 @@ |
155 | tx.commit() |
156 | |
157 | # Create the functions. |
158 | - for files, query, descr in zip((remote_funcs, remote_calls), |
159 | - (self.add_func_query, self.add_call_query), |
160 | - ('funcs', 'calls')): |
161 | + file_lists = (remote_funcs, remote_calls, remote_calls) |
162 | + queries = (self.add_func_query, self.add_internal_call_query, |
163 | + self.add_external_call_query) |
164 | + descrs = ('functions', 'internal calls', 'external calls') |
165 | + |
166 | + for files, query, descr in zip(file_lists, queries, descrs): |
167 | start = time() |
168 | for i, path in enumerate(files): |
169 | completed = int(100*float(i+1)/len(files)) |
170 | @@ -377,7 +395,7 @@ |
171 | Loop over all functions: increment the called-by count of their callees. |
172 | """ |
173 | for func in self.functions: |
174 | - for called in func.functions_i_call: |
175 | + for called, is_internal in func.functions_i_call: |
176 | called.number_calling_me += 1 |
177 | |
178 | def _rest_node_output_to_graph(self, rest_output): |
179 | @@ -442,8 +460,8 @@ |
180 | for_query=True) |
181 | for index in result: |
182 | q = ("START n=node({})" |
183 | - "MATCH n-[calls:calls]->(m)" |
184 | - "RETURN n.name, m.name").format(result[index][2]) |
185 | + "MATCH n-[r]->(m)" |
186 | + "RETURN n.name, m.name, type(r) = 'internal'").format(result[index][2]) |
187 | new_tx.append(q) |
188 | |
189 | logging.debug('exec') |
190 | @@ -454,13 +472,17 @@ |
191 | |
192 | for call_list in results: |
193 | if call_list: |
194 | - # call_list has element 0 being an arbitrary call this |
195 | - # function makes; element 0 of that call is the name of the |
196 | - # function itself. Think {{'orig', 'b'}, {'orig', 'c'}}. |
197 | - orig = call_list[0][0] |
198 | - # result['orig'] is [<Function>, ('callee1','callee2')] |
199 | - result[orig][1] |= set(list(zip(*call_list.elements))[1]) |
200 | - # recall: set union is denoted by | |
201 | + elements = call_list.elements |
202 | + # elements is a list of lists of form: |
203 | + # [[caller1, callee1, is_internal], |
204 | + # [caller1, callee2, is_internal], |
205 | + # ...] |
206 | + |
207 | + caller = elements[0][0] |
208 | + callees, is_internals = zip(*elements)[1:] |
209 | + |
210 | + # result[caller] is [<Function>, <set of callee, is_internal tuples>] |
211 | + result[caller][1] |= set(zip(callees, is_internals)) |
212 | |
213 | else: |
214 | # We don't have a parent database connection. |
215 | @@ -478,8 +500,8 @@ |
216 | named_function = lambda name: result[name][0] if name in result else None |
217 | |
218 | for function, calls, node_id in result.values(): |
219 | - what_i_call = [named_function(name) |
220 | - for name in calls |
221 | + what_i_call = [(named_function(name), is_internal) |
222 | + for name, is_internal in calls |
223 | if named_function(name) is not None] |
224 | function.functions_i_call = what_i_call |
225 | |
226 | @@ -707,7 +729,7 @@ |
227 | func_count, call_count = tx.commit()[0].elements[0] |
228 | |
229 | del_call_query = ('OPTIONAL MATCH (p:program {{name: "{}"}})' |
230 | - '-[:subject]->(f:func)-[c:calls]->()' |
231 | + '-[:subject]->(f:func)-[c]->()' |
232 | ' WITH c LIMIT 5000 DELETE c RETURN count(distinct(c))' |
233 | .format(program_name)) |
234 | |
235 | @@ -817,40 +839,32 @@ |
236 | result = self._db.query(q, returns=neo4jrestclient.Node) |
237 | return bool(result) |
238 | |
239 | - def check_function_exists(self, program_name, function_name): |
240 | - """ |
241 | - Execute query to check whether a function with the given name exists. |
242 | - We only check for functions which are children of a program with the |
243 | - given program_name. |
244 | - :param program_name: string name of the program within which to check |
245 | - :param function_name: string name of the function to check for existence |
246 | - :return: bool(names validate correctly, and function exists in program) |
247 | - """ |
248 | - if not validate_query(program_name): |
249 | - return False |
250 | - |
251 | - pmatch = '(:program {{name: "{}"}})'.format(program_name) |
252 | - fmatch = '(f:func {{name: "{}"}})'.format(function_name) |
253 | - # be explicit about index usage |
254 | - q = (' MATCH {}-[:subject]->{} USING INDEX f:func(name)' |
255 | - ' RETURN f LIMIT 1'.format(pmatch, fmatch)) |
256 | - |
257 | - # result will be an empty list if the function was not found |
258 | - result = self._db.query(q, returns=neo4jrestclient.Node) |
259 | - return bool(result) |
260 | - |
261 | def get_function_names(self, program_name, search=None, max_funcs=None): |
262 | """ |
263 | Execute query to retrieve a list of all functions in the program. |
264 | Any of the output names can be used verbatim in any SextantConnection |
265 | method which requires a function-name input. |
266 | - :param program_name: name of the program whose functions to retrieve |
267 | - :return: None if program_name doesn't exist in the remote database, |
268 | - a set of function-name strings otherwise. |
269 | + |
270 | + Return: |
271 | + None if program_name doesn't exist in the remote database, |
272 | + a set of function-name strings otherwise. |
273 | + Arguments: |
274 | + program_name: |
275 | + The name of the program to query. |
276 | + |
277 | + search: |
278 | + A string of form <name_match>:<file_match>, where at least |
279 | + one of name_match and file_match is provided, and each may be a |
280 | + comma separated list of strings containing wildcard '.*' |
281 | + sequences. |
282 | + |
283 | + max_funcs: |
284 | + An integer limiting the number of functions returned by this |
285 | + method. |
286 | """ |
287 | |
288 | if not validate_query(program_name): |
289 | - return set() |
290 | + return None |
291 | |
292 | limit = "LIMIT {}".format(max_funcs) if max_funcs else "" |
293 | |
294 | @@ -859,8 +873,8 @@ |
295 | ' RETURN f.name {}').format(program_name, limit) |
296 | else: |
297 | q = (' MATCH (:program {{name: "{}"}})-[:subject]->(f:func)' |
298 | - ' WHERE f.name =~ ".*{}.*" RETURN f.name {}' |
299 | - .format(program_name, search, limit)) |
300 | + ' {} RETURN f.name {}' |
301 | + .format(program_name, self.get_query('f', search), limit)) |
302 | return {func[0] for func in self._db.query(q)} |
303 | |
304 | @staticmethod |
305 | @@ -888,6 +902,7 @@ |
306 | etc. |
307 | |
308 | """ |
309 | + |
310 | if ':' in search: |
311 | func_subs, file_subs = search.split(':') |
312 | else: |
313 | @@ -928,49 +943,140 @@ |
314 | |
315 | return query_str |
316 | |
317 | - def get_all_functions_called(self, program_name, function_calling): |
318 | - """ |
319 | - Execute query to find all functions called by a function (indirectly). |
320 | - If the given function is not present in the program, returns None; |
321 | - likewise if the program_name does not exist. |
322 | - :param program_name: a string name of the program we wish to query under |
323 | - :param function_calling: string name of a function whose children to find |
324 | - :return: FunctionQueryResult, maximal subgraph rooted at function_calling |
325 | - """ |
326 | + def get_all_functions_called(self, program_name, function_calling, |
327 | + limit_internal, max_depth): |
328 | + """ |
329 | + Return the subtrees of the callgraph rooted at the specified functions. |
330 | + |
331 | + Optionally limit to a maximum call depth, or to only internal (same |
332 | + source file) calls. In the case of the latter, also include a single |
333 | + extra hop into external functions. |
334 | + |
335 | + If the function has no calls, it will be returned alone. |
336 | + |
337 | + Return: |
338 | + None if program_name doesn't exist in the remote database, |
339 | + a FunctionQueryResult containing the nodes and relationships |
340 | + otherwise. |
341 | + Arguments: |
342 | + program_name: |
343 | + The name of the program to query. |
344 | + |
345 | + function_calling: |
346 | + A string of form <name_match>:<file_match>, where at least |
347 | + one of name_match and file_match is provided, and each may be a |
348 | + comma separated list of strings containing wildcard '.*' |
349 | + sequences. Specifies the list of subtree roots. |
350 | + |
351 | + limit_internal: |
352 | + If true, only explore internal calls, but also add one extra |
353 | + level (above max_depth) along external calls. |
354 | + |
355 | + max_depth: |
356 | + A STRING containing an integer which will limit the depth |
357 | + of the subtrees. '0' corresponds to unlimited depth. |
358 | + """ |
359 | + |
360 | + if not validate_query(program_name): |
361 | + return None |
362 | + |
363 | + max_depth = max_depth or '1' |
364 | + |
365 | q = (' MATCH (p:program {{name: "{}"}})-[:subject]->(f:func) {}' |
366 | - ' MATCH (f)-[:calls]->(g:func) RETURN distinct f, g' |
367 | - .format(program_name, SextantConnection.get_query('f', function_calling))) |
368 | + ' MATCH (f)-[{}*0..{}]->(g:func)' |
369 | + .format(program_name, SextantConnection.get_query('f', function_calling), |
370 | + ':internal' if limit_internal=='true' else '', |
371 | + max_depth if int(max_depth) > 0 else '')) |
372 | + |
373 | + if limit_internal == 'true': |
374 | + q += (' WITH f, g MATCH (g)-[:external*0..1]->(h)' |
375 | + ' RETURN distinct f, h') |
376 | + else: |
377 | + q += ' RETURN distinct f, g' |
378 | |
379 | return self._execute_query(program_name, q) |
380 | |
381 | - def get_all_functions_calling(self, program_name, function_called): |
382 | - """ |
383 | - Execute query to find all functions which call a function (indirectly). |
384 | - If the given function is not present in the program, returns None; |
385 | - likewise if the program_name does not exist. |
386 | - :param program_name: a string name of the program we wish to query |
387 | - :param function_called: string name of a function whose parents to find |
388 | - :return: FunctionQueryResult, maximal connected subgraph with leaf function_called |
389 | - """ |
390 | + def get_all_functions_calling(self, program_name, function_called, |
391 | + limit_internal, max_depth): |
392 | + """ |
393 | + Return functions calling the specified functions. |
394 | + |
395 | + Optionally limit to a maximum call depth, or to only internal (same |
396 | + source file) calls. |
397 | + |
398 | + If the function is not called, return it alone. |
399 | + |
400 | + Return: |
401 | + None if program_name doesn't exist in the remote database, |
402 | + a FunctionQueryResult containing the nodes and relationships |
403 | + otherwise. |
404 | + Arguments: |
405 | + program_name: |
406 | + The name of the program to query. |
407 | + |
408 | + function_called: |
409 | + A string of form <name_match>:<file_match>, where at least |
410 | + one of name_match and file_match is provided, and each may be a |
411 | + comma separated list of strings containing wildcard '.*' |
412 | + sequences. Specifies the list of functions to match. |
413 | + |
414 | + limit_internal: |
415 | + If true, only explore internal calls. |
416 | + |
417 | + max_depth: |
418 | + A STRING containing an integer which will limit the depth |
419 | + of the subtrees. '0' corresponds to unlimited depth. |
420 | + """ |
421 | + |
422 | + if not validate_query(program_name): |
423 | + return None |
424 | + |
425 | + max_depth = max_depth or '1' |
426 | |
427 | q = (' MATCH (p:program {{name: "{}"}})-[:subject]->(g:func) {}' |
428 | - ' MATCH (f)-[:calls]->(g)' |
429 | + ' MATCH (f)-[{}*0..{}]->(g)' |
430 | ' RETURN distinct f, g') |
431 | - q = q.format(program_name, SextantConnection.get_query('g', function_called), program_name) |
432 | + q = q.format(program_name, SextantConnection.get_query('g', function_called), |
433 | + ':internal' if limit_internal=='true' else ':internal|external', |
434 | + max_depth if int(max_depth) > 0 else '') |
435 | |
436 | return self._execute_query(program_name, q) |
437 | |
438 | - def get_call_paths(self, program_name, function_calling, function_called): |
439 | - """ |
440 | - Execute query to find all possible routes between two specific nodes. |
441 | - If the given functions are not present in the program, returns None; |
442 | - ditto if the program_name does not exist. |
443 | - :param program_name: string program name |
444 | - :param function_calling: string |
445 | - :param function_called: string |
446 | - :return: FunctionQueryResult, the union of all subgraphs reachable by |
447 | - adding a source at function_calling and a sink at function_called. |
448 | - """ |
449 | + def get_call_paths(self, program_name, function_calling, function_called, |
450 | + limit_internal, max_depth): |
451 | + """ |
452 | + Return all call paths between the sets of functions specified by the |
453 | + search strings function_calling and function_called. |
454 | + |
455 | + Optionally limit to a maximum call depth, or to only internal (same |
456 | + source file) calls. |
457 | + |
458 | + Return: |
459 | + None if program_name doesn't exist in the remote database, |
460 | + a FunctionQueryResult object containing the nodes and relationships |
461 | + otherwise. |
462 | + Arguments: |
463 | + program_name: |
464 | + The name of the program to query. |
465 | + |
466 | + function_calling, function_called: |
467 | + A string of form <name_match>:<file_match>, where at least |
468 | + one of name_match and file_match is provided, and each may be a |
469 | + comma separated list of strings containing wildcard '.*' |
470 | + sequences. Specifies the list of functions to match. |
471 | + |
472 | + limit_internal: |
473 | + If true, only explore internal calls. |
474 | + |
475 | + max_depth: |
476 | + A STRING containing an integer which will limit the depth |
477 | + of the subtrees. '0' corresponds to unlimited depth. |
478 | + """ |
479 | + |
480 | + if not validate_query(program_name): |
481 | + return None |
482 | + |
483 | + max_depth = max_depth or '1' |
484 | |
485 | if not self.check_program_exists(program_name): |
486 | return None |
487 | @@ -981,18 +1087,25 @@ |
488 | q = (' MATCH (p:program {{name: "{}"}})' |
489 | ' MATCH (p)-[:subject]->(start:func) {} WITH start, p' |
490 | ' MATCH (p)-[:subject]->(end:func) {} WITH start, end' |
491 | - ' MATCH path=(start)-[:calls*]->(end)' |
492 | + ' MATCH path=(start)-[{}*0..{}]->(end)' |
493 | ' WITH DISTINCT nodes(path) AS result' |
494 | ' UNWIND result AS answer' |
495 | ' RETURN answer') |
496 | - q = q.format(program_name, start_q, end_q) |
497 | + q = q.format(program_name, start_q, end_q, ':internal' if limit_internal=='true' else '', |
498 | + max_depth if int(max_depth) > 0 else '') |
499 | return self._execute_query(program_name, q) |
500 | |
501 | def get_whole_program(self, program_name): |
502 | - """Execute query to find the entire program with a given name. |
503 | - If the program is not present in the remote database, returns None. |
504 | - :param: program_name: a string name of the program we wish to return. |
505 | - :return: a FunctionQueryResult consisting of the program graph. |
506 | + """ |
507 | + Return the full call graph of the program. |
508 | + |
509 | + Return: |
510 | + None if program_name doesn't exist in the remote database, |
511 | + a FunctionQueryResult object containing all of the program nodes |
512 | + and calls otherwise. |
513 | + Arguments: |
514 | + program_name: |
515 | + The name of the program to query. |
516 | """ |
517 | |
518 | if not self.check_program_exists(program_name): |
519 | @@ -1002,7 +1115,9 @@ |
520 | ' RETURN (f)'.format(program_name)) |
521 | return self._execute_query(program_name, q) |
522 | |
523 | - def get_shortest_path_between_functions(self, program_name, function_calling, function_called): |
524 | + def get_shortest_path_between_functions(self, program_name, |
525 | + function_calling, function_called, |
526 | + limit_internal, max_depth): |
527 | """ |
528 | Execute query to get a single, shortest, path between two functions. |
529 | :param program_name: string name of the program we wish to search under |
530 | @@ -1010,6 +1125,39 @@ |
531 | :param func2: the name of the function at which to terminate the path |
532 | :return: FunctionQueryResult shortest path between func1 and func2. |
533 | """ |
534 | + """ |
535 | + Return all shortest paths between the sets of functions specified. |
536 | + |
537 | + Optionally limit to a maximum call depth, or to only internal (same |
538 | + source file) calls. |
539 | + |
540 | + Return: |
541 | + None if program_name doesn't exist in the remote database, |
542 | + a FunctionQueryResult object containing the nodes and relationships |
543 | + otherwise. |
544 | + Arguments: |
545 | + program_name: |
546 | + The name of the program to query. |
547 | + |
548 | + function_calling, function_called: |
549 | + A string of form <name_match>:<file_match>, where at least |
550 | + one of name_match and file_match is provided, and each may be a |
551 | + comma separated list of strings containing wildcard '.*' |
552 | + sequences. Specifies the list of functions to match. |
553 | + |
554 | + limit_internal: |
555 | + If true, only explore internal calls. |
556 | + |
557 | + max_depth: |
558 | + A STRING containing an integer which will limit the depth |
559 | + of the subtrees. '0' corresponds to unlimited depth. |
560 | + """ |
561 | + |
562 | + if not self.check_program_exists(program_name): |
563 | + return None |
564 | + |
565 | + max_depth = max_depth or '1' |
566 | + |
567 | if not self.check_program_exists(program_name): |
568 | return None |
569 | |
570 | @@ -1019,10 +1167,11 @@ |
571 | q = (' MATCH (p:program {{name: "{}"}})' |
572 | ' MATCH (p)-[:subject]->(start:func) {} WITH start, p' |
573 | ' MATCH (p)-[:subject]->(end:func) {} WITH start, end' |
574 | - ' MATCH path=shortestPath((start)-[:calls*]->(end))' |
575 | + ' MATCH path=allShortestPaths((start)-[{}*..{}]->(end))' |
576 | ' UNWIND nodes(path) AS answer' |
577 | ' RETURN answer') |
578 | - q = q.format(program_name, start_q, end_q) |
579 | + q = q.format(program_name, start_q, end_q, ':internal' if limit_internal=='true' else '', |
580 | + max_depth if int(max_depth) > 0 else '') |
581 | |
582 | return self._execute_query(program_name, q) |
583 | |
584 | |
585 | === modified file 'src/sextant/export.py' |
586 | --- src/sextant/export.py 2014-11-19 10:32:34 +0000 |
587 | +++ src/sextant/export.py 2014-11-25 10:17:59 +0000 |
588 | @@ -71,9 +71,10 @@ |
589 | if func_called == func: |
590 | functions_called.remove(func_called) |
591 | |
592 | - for func_called in func.functions_i_call: |
593 | + for func_called, is_internal in func.functions_i_call: |
594 | if not (suppress_common_nodes and func_called.is_common): |
595 | - output_str += ' "{}" -> "{}"\n'.format(func.name, func_called.name) |
596 | + color = 'black' if is_internal else 'red' |
597 | + output_str += ' "{}" -> "{}" [color={}]\n'.format(func.name, func_called.name, color) |
598 | |
599 | output_str += '}' |
600 | return output_str |
601 | @@ -140,18 +141,21 @@ |
602 | </node>\n""".format(func.name, 20, len(display_func)*8, colour, display_func) |
603 | |
604 | functions_called = func.functions_i_call |
605 | + print(functions_called) |
606 | if remove_self_calls is True: |
607 | #remove calls where a function calls itself |
608 | - for func_called in functions_called: |
609 | + for func_called, is_internal, in functions_called: |
610 | if func_called == func: |
611 | functions_called.remove(func_called) |
612 | |
613 | - for callee in functions_called: |
614 | + for callee, is_internal in functions_called: |
615 | + print(callee, is_internal) |
616 | + color = "#000000" if is_internal else "#ff0000" |
617 | if callee not in commonly_called: |
618 | if not(suppress_common_nodes and callee.is_common): |
619 | output_str += """<edge source="{}" target="{}"> <data key="d9"> |
620 | <y:PolyLineEdge> |
621 | - <y:LineStyle color="#000000" type="line" width="1.0"/> |
622 | + <y:LineStyle color="#ff0000" type="line" width="1.0"/> |
623 | <y:Arrows source="none" target="standard"/> |
624 | <y:BendStyle smoothed="false"/> |
625 | </y:PolyLineEdge> |
626 | |
627 | === modified file 'src/sextant/objdump_parser.py' |
628 | --- src/sextant/objdump_parser.py 2014-11-21 15:19:15 +0000 |
629 | +++ src/sextant/objdump_parser.py 2014-11-25 10:17:59 +0000 |
630 | @@ -43,11 +43,15 @@ |
631 | function_ptr_count: |
632 | The number of function pointers that have been detected. |
633 | _known_functions: |
634 | - A set of the names of functions that have been |
635 | - parsed - used to avoid registering a function multiple times. |
636 | + A dict of the names of functions that have been |
637 | + parsed - used to avoid registering a function multiple times |
638 | + and to label calls as internal/external. |
639 | _partial_functions: |
640 | A set of functions whose names we have seen but whose source |
641 | files we don't yet know. |
642 | + _partial_calls: |
643 | + A set of the (caller, callee) tuples representing calls between |
644 | + a _partial_function and another function. |
645 | |
646 | """ |
647 | def __init__(self, file_path, file_object=None, |
648 | @@ -106,16 +110,18 @@ |
649 | self.function_ptr_count = 0 |
650 | |
651 | # Avoid adding duplicate functions. |
652 | - self._known_functions = set() |
653 | + self._known_functions = dict() |
654 | + self._known_calls = set() |
655 | # Set of partially-parsed functions. |
656 | self._partial_functions = set() |
657 | + self._partial_calls = set() |
658 | |
659 | # By default print information to stdout. |
660 | def print_func(name, typ, source='unknown'): |
661 | - print('func {:25}{:15}{}'.format(name, typ, source)) |
662 | + print('func {:25} {:15}{}'.format(name, typ, source)) |
663 | |
664 | - def print_call(caller, callee): |
665 | - print('call {:25}{:25}'.format(caller, callee)) |
666 | + def print_call(caller, callee, is_internal): |
667 | + print('call {:25} {:25}'.format(caller, callee)) |
668 | |
669 | def print_started(parser): |
670 | print('parse started: {}[{}]'.format(self.path, ', '.join(self.sections))) |
671 | @@ -148,6 +154,7 @@ |
672 | self._partial_functions.add(name) |
673 | elif source == 'unknown': |
674 | # Manually adding a stub function. |
675 | + self._known_functions[name] = source |
676 | self.add_function(name, 'stub', source) |
677 | self.function_count += 1 |
678 | elif name not in self._known_functions: |
679 | @@ -160,7 +167,7 @@ |
680 | except KeyError: |
681 | pass |
682 | |
683 | - self._known_functions.add(name) |
684 | + self._known_functions[name] = source |
685 | self.add_function(name, 'normal', source) |
686 | self.function_count += 1 |
687 | |
688 | @@ -168,15 +175,50 @@ |
689 | """ |
690 | Add a function pointer. |
691 | """ |
692 | - self.add_function(name, 'pointer') |
693 | + self.add_function(name, 'pointer', 'unknown') |
694 | self.function_count += 1 |
695 | |
696 | - def _add_call(self, caller, callee): |
697 | + def _add_call(self, caller, callee, force=False): |
698 | """ |
699 | Add a function call from caller to callee. |
700 | """ |
701 | - self.add_call(caller, callee) |
702 | - self.call_count += 1 |
703 | + if (caller, callee) in self._known_calls: |
704 | + return |
705 | + |
706 | + try: |
707 | + files = self._known_functions |
708 | + is_internal = (callee.startswith('func_ptr_') |
709 | + or (files[caller] != 'unknown' and files[caller] == files[callee])) |
710 | + self.add_call(caller, callee, is_internal) |
711 | + self._known_calls.add((caller, callee)) |
712 | + self.call_count += 1 |
713 | + except KeyError: |
714 | + if force: |
715 | + self._add_function(callee, 'unknown') |
716 | + self.add_call(caller, callee, False) |
717 | + self._known_calls.add((caller, callee)) |
718 | + self.call_count += 1 |
719 | + print(caller, callee) |
720 | + else: |
721 | + self._partial_calls.add((caller, callee)) |
722 | + |
723 | + @staticmethod |
724 | + def clean_id(function_identifier): |
725 | + """ |
726 | + Clean the funciton identifier string. |
727 | + """ |
728 | + # IOS builds add a __be_ (big endian) prefix to all functions, |
729 | + # get rid of it if it is there, |
730 | + if function_identifier.startswith('__be_'): |
731 | + function_identifier = function_identifier[len('__be_'):] |
732 | + |
733 | + # Some functions look like <identifier>. or <identifier>..<digit> |
734 | + # - get rid of the extra bits here: |
735 | + if '.' in function_identifier: |
736 | + function_identifier = function_identifier.split('.')[0] |
737 | + |
738 | + return function_identifier |
739 | + |
740 | |
741 | def parse(self): |
742 | """ |
743 | @@ -193,6 +235,10 @@ |
744 | if to_add: |
745 | file_line = line.startswith('/') |
746 | source = line.split(':')[0] if file_line else None |
747 | + if source: |
748 | + # Prune out the relative parts of the filepath. |
749 | + source = source.rsplit('./', 1)[-1] |
750 | + |
751 | self._add_function(current_function, source) |
752 | to_add = False |
753 | |
754 | @@ -207,16 +253,15 @@ |
755 | self.section_count += 1 |
756 | |
757 | elif in_section: |
758 | + |
759 | + |
760 | if line.endswith('>:\n'): |
761 | # '<address> <<function_identifier>>:\n' |
762 | # with <function_identifier> of form: |
763 | # <function_name>[@plt] |
764 | function_identifier = line.split('<')[-1].split('>')[0] |
765 | |
766 | - # IOS builds add a __be_ (big endian) prefix to all functions, |
767 | - # get rid of it if it is there, |
768 | - if function_identifier.startswith('__be_'): |
769 | - function_identifier = function_identifier[len('__be_'):] |
770 | + function_identifier = self.clean_id(function_identifier) |
771 | |
772 | if '@' in function_identifier: |
773 | # Of form <function name>@<other stuff>. |
774 | @@ -227,6 +272,10 @@ |
775 | # Flag function - we look for source on the next line. |
776 | to_add = True |
777 | |
778 | + # If we have come to a new current_function, then we can |
779 | + # forget about the calls we knew about from the last. |
780 | + self._known_calls = set() |
781 | + |
782 | elif 'call ' in line or 'callq ' in line: |
783 | # WHITESPACE to prevent picking up function names |
784 | # containing 'call' |
785 | @@ -244,8 +293,8 @@ |
786 | # from which we extract name |
787 | callee_is_ptr = False |
788 | function_identifier = callee_info.lstrip('<').rstrip('>\n') |
789 | - if function_identifier.startswith('__be_'): |
790 | - function_identifier = function_identifier[len('__be_'):] |
791 | + |
792 | + function_identifier = self.clean_id(function_identifier) |
793 | |
794 | if '@' in function_identifier: |
795 | callee = function_identifier.split('@')[0] |
796 | @@ -268,7 +317,8 @@ |
797 | |
798 | for name in self._partial_functions: |
799 | self._add_function(name, 'unknown') |
800 | - |
801 | + for call in sorted(self._partial_calls, key=lambda el: el[0]): |
802 | + self._add_call(*call, force=True) |
803 | |
804 | self.finished() |
805 | |
806 | |
807 | === added file 'src/sextant/test_db.py' |
808 | --- src/sextant/test_db.py 1970-01-01 00:00:00 +0000 |
809 | +++ src/sextant/test_db.py 2014-11-25 10:17:59 +0000 |
810 | @@ -0,0 +1,97 @@ |
811 | +#!/usr/bin/python |
812 | +# ----------------------------------------- |
813 | +# Sextant |
814 | +# Copyright 2014, Ensoft Ltd. |
815 | +# Author: Patrick Stevens, James Harkin |
816 | +# ----------------------------------------- |
817 | +#Testing module |
818 | + |
819 | +import unittest |
820 | + |
821 | +import db_api |
822 | +import update_db |
823 | + |
824 | +PNAME = 'tester-parser_test' |
825 | +NORMAL = {'main', 'normal', 'wierd$name', 'duplicates'} |
826 | + |
827 | + |
828 | +class TestFunctionQueryResults(unittest.TestCase): |
829 | + @classmethod |
830 | + def setUpClass(cls): |
831 | + # we need to set up the remote database by using the neo4j_input_api |
832 | + cls.remote_url = 'http://ensoft-sandbox:7474' |
833 | + cls.connection = db_api.SextantConnection('ensoft-sandbox', 7474) |
834 | + |
835 | + update_db.upload_program |
836 | + update_db.upload_program(cls.connection, 'tester', 'test_resources/parser_test', |
837 | + program_name=None, not_object_file=False, add_file_paths=True) |
838 | + |
839 | + @classmethod |
840 | + def tearDownClass(cls): |
841 | + cls.connection.delete_program('tester-parser_test') |
842 | + cls.connection.close() |
843 | + |
844 | + def test_get_function_names(self): |
845 | + get_names = self.connection.get_function_names |
846 | + # Test getting all names |
847 | + names = get_names(PNAME) |
848 | + # Test file wildcard search |
849 | + parser_names = get_names(PNAME, search=':.*parser_test.c') |
850 | + |
851 | + self.assertTrue(names.issuperset(NORMAL)) |
852 | + self.assertEquals(len(names), 24) |
853 | + self.assertEquals(parser_names, {u'main', u'normal', u'duplicates', u'wierd$name'}) |
854 | + |
855 | + # Test the wildcard matching |
856 | + search = self.connection.get_function_names(PNAME, search='.*libc.*') |
857 | + search_exp = {u'__libc_csu_init', u'__libc_csu_fini', u'__libc_start_main'} |
858 | + |
859 | + self.assertEquals(search, search_exp) |
860 | + |
861 | + # Test the limiting |
862 | + too_many = self.connection.get_function_names(PNAME, max_funcs=3) |
863 | + self.assertEquals(len(too_many), 3) |
864 | + |
865 | + # Test empty for non-existant program |
866 | + self.assertFalse(self.connection.get_function_names('blah blah blah')) |
867 | + |
868 | + def test_get_all_functions_called(self): |
869 | + get_fns = self.connection.get_all_functions_called |
870 | + |
871 | + for depth, num in zip([0, 1, 2, 3], [8, 3, 8, 8]): |
872 | + result = get_fns(PNAME, 'main', "false", str(depth)).functions |
873 | + self.assertEquals(len(result), num, str(result)) |
874 | + |
875 | + for depth, num in zip([0, 1, 2, 3], [8, 4, 8, 8]): |
876 | + # Limit to internal functions |
877 | + # TODO this isn't a great test - need greater call depth |
878 | + result = get_fns(PNAME, 'main', "true", str(depth)).functions |
879 | + self.assertEquals(len(result), num) |
880 | + |
881 | + def test_get_all_functions_calling(self): |
882 | + get_fns = self.connection.get_all_functions_calling |
883 | + |
884 | + for depth, num in zip(range(3), [4, 3, 4, 4]): |
885 | + result = get_fns(PNAME, 'printf', limit_internal="false", max_depth=str(depth)).functions |
886 | + self.assertEquals(len(result), num) |
887 | + |
888 | + for depth, num in zip(range(3), [1, 1, 1, 1]): |
889 | + result = get_fns(PNAME, 'printf', limit_internal="true", max_depth=str(depth)).functions |
890 | + self.assertEquals(len(result), num) |
891 | + |
892 | + def test_get_all_paths_between(self): |
893 | + get_paths = self.connection.get_call_paths |
894 | + |
895 | + result = {f.name for f in get_paths(PNAME, 'main', 'wierd$name', 'false', '0').functions} |
896 | + exp = {'main', 'normal', 'duplicates', 'wierd$name'} |
897 | + self.assertEquals(result, exp) |
898 | + |
899 | + def test_get_shortest_paths_between(self): |
900 | + get_paths = self.connection.get_shortest_path_between_functions |
901 | + |
902 | + result = {f.name for f in get_paths(PNAME, 'main', 'wierd$name', 'false', '0').functions} |
903 | + exp = {u'main', u'normal', u'wierd$name'} |
904 | + self.assertEquals(result, exp) |
905 | + |
906 | +if __name__ == '__main__': |
907 | + unittest.main() |
908 | |
909 | === removed file 'src/sextant/test_db_api.py' |
910 | --- src/sextant/test_db_api.py 2014-10-16 15:26:47 +0000 |
911 | +++ src/sextant/test_db_api.py 1970-01-01 00:00:00 +0000 |
912 | @@ -1,275 +0,0 @@ |
913 | -#!/usr/bin/python |
914 | -# ----------------------------------------- |
915 | -# Sextant |
916 | -# Copyright 2014, Ensoft Ltd. |
917 | -# Author: Patrick Stevens, James Harkin |
918 | -# ----------------------------------------- |
919 | -#Testing module |
920 | - |
921 | -import unittest |
922 | - |
923 | -from db_api import Function |
924 | -from db_api import FunctionQueryResult |
925 | -from db_api import SextantConnection |
926 | -from db_api import validate_query |
927 | - |
928 | - |
929 | -class TestFunctionQueryResults(unittest.TestCase): |
930 | - @classmethod |
931 | - def setUpClass(cls): |
932 | - # we need to set up the remote database by using the neo4j_input_api |
933 | - cls.remote_url = 'http://ensoft-sandbox:7474' |
934 | - |
935 | - cls.setter_connection = SextantConnection('ensoft-sandbox', 7474) |
936 | - |
937 | - cls.program_1_name = 'testprogram' |
938 | - cls.one_node_program_name = 'testprogram1' |
939 | - cls.empty_program_name = 'testprogramblank' |
940 | - |
941 | - # if anything failed before, delete programs now |
942 | - cls.setter_connection.delete_program(cls.program_1_name) |
943 | - cls.setter_connection.delete_program(cls.one_node_program_name) |
944 | - cls.setter_connection.delete_program(cls.empty_program_name) |
945 | - |
946 | - |
947 | - cls.upload_program = cls.setter_connection.new_program(cls.program_1_name) |
948 | - cls.upload_program.add_function('func1') |
949 | - cls.upload_program.add_function('func2') |
950 | - cls.upload_program.add_function('func3') |
951 | - cls.upload_program.add_function('func4') |
952 | - cls.upload_program.add_function('func5') |
953 | - cls.upload_program.add_function('func6') |
954 | - cls.upload_program.add_function('func7') |
955 | - cls.upload_program.add_call('func1', 'func2') |
956 | - cls.upload_program.add_call('func1', 'func4') |
957 | - cls.upload_program.add_call('func2', 'func1') |
958 | - cls.upload_program.add_call('func2', 'func4') |
959 | - cls.upload_program.add_call('func3', 'func5') |
960 | - cls.upload_program.add_call('func4', 'func4') |
961 | - cls.upload_program.add_call('func4', 'func5') |
962 | - cls.upload_program.add_call('func5', 'func1') |
963 | - cls.upload_program.add_call('func5', 'func2') |
964 | - cls.upload_program.add_call('func5', 'func3') |
965 | - cls.upload_program.add_call('func6', 'func7') |
966 | - |
967 | - cls.upload_program.commit() |
968 | - |
969 | - cls.upload_one_node_program = cls.setter_connection.new_program(cls.one_node_program_name) |
970 | - cls.upload_one_node_program.add_function('lonefunc') |
971 | - |
972 | - cls.upload_one_node_program.commit() |
973 | - |
974 | - cls.upload_empty_program = cls.setter_connection.new_program(cls.empty_program_name) |
975 | - |
976 | - cls.upload_empty_program.commit() |
977 | - |
978 | - cls.getter_connection = cls.setter_connection |
979 | - |
980 | - |
981 | - @classmethod |
982 | - def tearDownClass(cls): |
983 | - cls.setter_connection.delete_program(cls.upload_program.program_name) |
984 | - cls.setter_connection.delete_program(cls.upload_one_node_program.program_name) |
985 | - cls.setter_connection.delete_program(cls.upload_empty_program.program_name) |
986 | - |
987 | - cls.setter_connection.close() |
988 | - del(cls.setter_connection) |
989 | - |
990 | - def test_17_get_call_paths(self): |
991 | - reference1 = FunctionQueryResult(parent_db=None, program_name=self.program_1_name) |
992 | - reference1.functions = [Function(self.program_1_name, 'func1'), Function(self.program_1_name, 'func2'), |
993 | - Function(self.program_1_name, 'func3'), |
994 | - Function(self.program_1_name, 'func4'), Function(self.program_1_name, 'func5')] |
995 | - reference1.functions[0].functions_i_call = reference1.functions[1:4:2] # func1 calls func2, func4 |
996 | - reference1.functions[1].functions_i_call = reference1.functions[0:4:3] # func2 calls func1, func4 |
997 | - reference1.functions[2].functions_i_call = [reference1.functions[4]] # func3 calls func5 |
998 | - reference1.functions[3].functions_i_call = reference1.functions[3:5] # func4 calls func4, func5 |
999 | - reference1.functions[4].functions_i_call = reference1.functions[0:3] # func5 calls func1, func2, func3 |
1000 | - self.assertEquals(reference1, self.getter_connection.get_call_paths(self.program_1_name, 'func1', 'func2')) |
1001 | - self.assertIsNone(self.getter_connection.get_call_paths('not a prog', 'func1', 'func2')) # shouldn't validation |
1002 | - self.assertIsNone(self.getter_connection.get_call_paths('notaprogram', 'func1', 'func2')) |
1003 | - self.assertIsNone(self.getter_connection.get_call_paths(self.program_1_name, 'notafunc', 'func2')) |
1004 | - self.assertIsNone(self.getter_connection.get_call_paths(self.program_1_name, 'func1', 'notafunc')) |
1005 | - |
1006 | - def test_02_get_whole_program(self): |
1007 | - reference = FunctionQueryResult(parent_db=None, program_name=self.program_1_name) |
1008 | - reference.functions = [Function(self.program_1_name, 'func1'), Function(self.program_1_name, 'func2'), |
1009 | - Function(self.program_1_name, 'func3'), |
1010 | - Function(self.program_1_name, 'func4'), Function(self.program_1_name, 'func5'), |
1011 | - Function(self.program_1_name, 'func6'), Function(self.program_1_name, 'func7')] |
1012 | - reference.functions[0].functions_i_call = reference.functions[1:4:2] # func1 calls func2, func4 |
1013 | - reference.functions[1].functions_i_call = reference.functions[0:4:3] # func2 calls func1, func4 |
1014 | - reference.functions[2].functions_i_call = [reference.functions[4]] # func3 calls func5 |
1015 | - reference.functions[3].functions_i_call = reference.functions[3:5] # func4 calls func4, func5 |
1016 | - reference.functions[4].functions_i_call = reference.functions[0:3] # func5 calls func1, func2, func3 |
1017 | - reference.functions[5].functions_i_call = [reference.functions[6]] # func6 calls func7 |
1018 | - |
1019 | - |
1020 | - self.assertEqual(reference, self.getter_connection.get_whole_program(self.program_1_name)) |
1021 | - self.assertIsNone(self.getter_connection.get_whole_program('nottherightprogramname')) |
1022 | - |
1023 | - def test_03_get_whole_one_node_program(self): |
1024 | - reference = FunctionQueryResult(parent_db=None, program_name=self.one_node_program_name) |
1025 | - reference.functions = [Function(self.one_node_program_name, 'lonefunc')] |
1026 | - |
1027 | - self.assertEqual(reference, self.getter_connection.get_whole_program(self.one_node_program_name)) |
1028 | - |
1029 | - def test_04_get_whole_empty_program(self): |
1030 | - reference = FunctionQueryResult(parent_db=None, program_name=self.empty_program_name) |
1031 | - reference.functions = [] |
1032 | - |
1033 | - self.assertEqual(reference, self.getter_connection.get_whole_program(self.empty_program_name)) |
1034 | - |
1035 | - def test_05_get_function_names(self): |
1036 | - reference = {'func1', 'func2', 'func3', 'func4', 'func5', 'func6', 'func7'} |
1037 | - self.assertEqual(reference, self.getter_connection.get_function_names(self.program_1_name)) |
1038 | - |
1039 | - def test_06_get_function_names_one_node_program(self): |
1040 | - reference = {'lonefunc'} |
1041 | - self.assertEqual(reference, self.getter_connection.get_function_names(self.one_node_program_name)) |
1042 | - |
1043 | - def test_07_get_function_names_empty_program(self): |
1044 | - reference = set() |
1045 | - self.assertEqual(reference, self.getter_connection.get_function_names(self.empty_program_name)) |
1046 | - |
1047 | - def test_09_validation_is_used(self): |
1048 | - self.assertFalse(self.getter_connection.get_function_names('not alphanumeric')) |
1049 | - self.assertFalse(self.getter_connection.get_whole_program('not alphanumeric')) |
1050 | - self.assertFalse(self.getter_connection.check_program_exists('not alphanumeric')) |
1051 | - self.assertFalse(self.getter_connection.check_function_exists('not alphanumeric', 'alpha')) |
1052 | - self.assertFalse(self.getter_connection.check_function_exists('alpha', 'not alpha')) |
1053 | - self.assertFalse(self.getter_connection.get_all_functions_called('alphaprogram', 'not alpha function')) |
1054 | - self.assertFalse(self.getter_connection.get_all_functions_called('not alpha program', 'alphafunction')) |
1055 | - self.assertFalse(self.getter_connection.get_all_functions_calling('not alpha program', 'alphafunction')) |
1056 | - self.assertFalse(self.getter_connection.get_all_functions_calling('alphaprogram', 'not alpha function')) |
1057 | - self.assertFalse(self.getter_connection.get_call_paths('not alpha program','alphafunc1', 'alphafunc2')) |
1058 | - self.assertFalse(self.getter_connection.get_call_paths('alphaprogram','not alpha func 1', 'alphafunc2')) |
1059 | - self.assertFalse(self.getter_connection.get_call_paths('alphaprogram','alphafunc1', 'not alpha func 2')) |
1060 | - |
1061 | - def test_08_get_program_names(self): |
1062 | - reference = {self.program_1_name, self.one_node_program_name, self.empty_program_name} |
1063 | - self.assertTrue(reference.issubset(self.getter_connection.get_program_names())) |
1064 | - |
1065 | - |
1066 | - def test_11_get_all_functions_called(self): |
1067 | - reference1 = FunctionQueryResult(parent_db=None, program_name=self.program_1_name) # this will be the 1,2,3,4,5 component |
1068 | - reference2 = FunctionQueryResult(parent_db=None, program_name=self.program_1_name) # this will be the 6,7 component |
1069 | - reference3 = FunctionQueryResult(parent_db=None, program_name=self.program_1_name) # this will be the 7 component |
1070 | - reference1.functions = [Function(self.program_1_name, 'func1'), Function(self.program_1_name, 'func2'), |
1071 | - Function(self.program_1_name, 'func3'), |
1072 | - Function(self.program_1_name, 'func4'), Function(self.program_1_name, 'func5')] |
1073 | - reference2.functions = [Function(self.program_1_name, 'func6'), Function(self.program_1_name, 'func7')] |
1074 | - reference3.functions = [] |
1075 | - |
1076 | - reference1.functions[0].functions_i_call = reference1.functions[1:4:2] # func1 calls func2, func4 |
1077 | - reference1.functions[1].functions_i_call = reference1.functions[0:4:3] # func2 calls func1, func4 |
1078 | - reference1.functions[2].functions_i_call = [reference1.functions[4]] # func3 calls func5 |
1079 | - reference1.functions[3].functions_i_call = reference1.functions[3:5] # func4 calls func4, func5 |
1080 | - reference1.functions[4].functions_i_call = reference1.functions[0:3] # func5 calls func1, func2, func3 |
1081 | - |
1082 | - reference2.functions[0].functions_i_call = [reference2.functions[1]] |
1083 | - |
1084 | - self.assertEquals(reference1, self.getter_connection.get_all_functions_called(self.program_1_name, 'func1')) |
1085 | - self.assertEquals(reference1, self.getter_connection.get_all_functions_called(self.program_1_name, 'func2')) |
1086 | - self.assertEquals(reference1, self.getter_connection.get_all_functions_called(self.program_1_name, 'func3')) |
1087 | - self.assertEquals(reference1, self.getter_connection.get_all_functions_called(self.program_1_name, 'func4')) |
1088 | - self.assertEquals(reference1, self.getter_connection.get_all_functions_called(self.program_1_name, 'func5')) |
1089 | - |
1090 | - self.assertEquals(reference2, self.getter_connection.get_all_functions_called(self.program_1_name, 'func6')) |
1091 | - |
1092 | - self.assertEquals(reference3, self.getter_connection.get_all_functions_called(self.program_1_name, 'func7')) |
1093 | - |
1094 | - self.assertIsNone(self.getter_connection.get_all_functions_called(self.program_1_name, 'nottherightfunction')) |
1095 | - self.assertIsNone(self.getter_connection.get_all_functions_called('nottherightprogram', 'func2')) |
1096 | - |
1097 | - def test_12_get_all_functions_called_1(self): |
1098 | - reference1 = FunctionQueryResult(parent_db=None, program_name=self.one_node_program_name) |
1099 | - reference1.functions = [] |
1100 | - |
1101 | - d=self.getter_connection.get_all_functions_called(self.one_node_program_name, 'lonefunc') |
1102 | - self.assertEquals(reference1, self.getter_connection.get_all_functions_called(self.one_node_program_name, |
1103 | - 'lonefunc')) |
1104 | - self.assertIsNone(self.getter_connection.get_all_functions_called(self.one_node_program_name, |
1105 | - 'not the right function')) |
1106 | - self.assertIsNone(self.getter_connection.get_all_functions_called('not the right program', 'lonefunc')) |
1107 | - |
1108 | - def test_13_get_all_functions_called_blank(self): |
1109 | - reference1 = FunctionQueryResult(parent_db=None, program_name=self.empty_program_name) |
1110 | - reference1.functions = [] |
1111 | - |
1112 | - self.assertIsNone(self.getter_connection.get_all_functions_called(self.empty_program_name, |
1113 | - 'not the right function')) |
1114 | - |
1115 | - def test_14_get_all_functions_calling(self): |
1116 | - reference1 = FunctionQueryResult(parent_db=None, program_name=self.program_1_name) # this will be the 1,2,3,4,5 component |
1117 | - reference2 = FunctionQueryResult(parent_db=None, program_name=self.program_1_name) # this will be the 6,7 component |
1118 | - reference3 = FunctionQueryResult(parent_db=None, program_name=self.program_1_name) # this will be the 7 component |
1119 | - reference1.functions = [Function(self.program_1_name, 'func1'), Function(self.program_1_name, 'func2'), |
1120 | - Function(self.program_1_name, 'func3'), |
1121 | - Function(self.program_1_name, 'func4'), Function(self.program_1_name, 'func5')] |
1122 | - |
1123 | - reference1.functions[0].functions_i_call = reference1.functions[1:4:2] # func1 calls func2, func4 |
1124 | - reference1.functions[1].functions_i_call = reference1.functions[0:4:3] # func2 calls func1, func4 |
1125 | - reference1.functions[2].functions_i_call = [reference1.functions[4]] # func3 calls func5 |
1126 | - reference1.functions[3].functions_i_call = reference1.functions[3:5] # func4 calls func4, func5 |
1127 | - reference1.functions[4].functions_i_call = reference1.functions[0:3] # func5 calls func1, func2, func3 |
1128 | - |
1129 | - reference2.functions = [Function(self.program_1_name, 'func6'), Function(self.program_1_name, 'func7')] |
1130 | - |
1131 | - reference2.functions[0].functions_i_call = [reference2.functions[1]] |
1132 | - |
1133 | - reference3.functions = [Function(self.program_1_name, 'func6')] |
1134 | - |
1135 | - reference3.functions = [] |
1136 | - |
1137 | - self.assertEquals(reference1, self.getter_connection.get_all_functions_calling(self.program_1_name, 'func1')) |
1138 | - self.assertEquals(reference1, self.getter_connection.get_all_functions_calling(self.program_1_name, 'func2')) |
1139 | - self.assertEquals(reference1, self.getter_connection.get_all_functions_calling(self.program_1_name, 'func3')) |
1140 | - self.assertEquals(reference1, self.getter_connection.get_all_functions_calling(self.program_1_name, 'func4')) |
1141 | - self.assertEquals(reference1, self.getter_connection.get_all_functions_calling(self.program_1_name, 'func5')) |
1142 | - |
1143 | - self.assertEquals(reference2, self.getter_connection.get_all_functions_calling(self.program_1_name,'func7')) |
1144 | - |
1145 | - self.assertEquals(reference3, self.getter_connection.get_all_functions_calling(self.program_1_name, 'func6')) |
1146 | - |
1147 | - self.assertIsNone(self.getter_connection.get_all_functions_calling(self.program_1_name, 'nottherightfunction')) |
1148 | - self.assertIsNone(self.getter_connection.get_all_functions_calling('nottherightprogram', 'func2')) |
1149 | - |
1150 | - def test_15_get_all_functions_calling_one_node_prog(self): |
1151 | - reference1 = FunctionQueryResult(parent_db=None, program_name=self.one_node_program_name) |
1152 | - reference1.functions = [] |
1153 | - self.assertEquals(reference1, self.getter_connection.get_all_functions_calling(self.one_node_program_name, |
1154 | - 'lonefunc')) |
1155 | - self.assertIsNone(self.getter_connection.get_all_functions_calling(self.one_node_program_name, |
1156 | - 'not the right function')) |
1157 | - self.assertIsNone(self.getter_connection.get_all_functions_calling('not the right program', 'lonefunc')) |
1158 | - |
1159 | - def test_16_get_all_functions_calling_blank_prog(self): |
1160 | - reference1 = FunctionQueryResult(parent_db=None, program_name=self.empty_program_name) |
1161 | - reference1.functions=[] |
1162 | - |
1163 | - self.assertIsNone(self.getter_connection.get_all_functions_called(self.empty_program_name, |
1164 | - 'not the right function')) |
1165 | - |
1166 | - |
1167 | - |
1168 | - def test_18_get_call_paths_between_two_functions_one_node_prog(self): |
1169 | - reference = FunctionQueryResult(parent_db=None, program_name=self.one_node_program_name) |
1170 | - reference.functions = [] # that is, reference is the empty program with name self.one_node_program_name |
1171 | - |
1172 | - self.assertEquals(self.getter_connection.get_call_paths(self.one_node_program_name, 'lonefunc', 'lonefunc'), |
1173 | - reference) |
1174 | - self.assertIsNone(self.getter_connection.get_call_paths(self.one_node_program_name, 'lonefunc', 'notafunc')) |
1175 | - self.assertIsNone(self.getter_connection.get_call_paths(self.one_node_program_name, 'notafunc', 'notafunc')) |
1176 | - |
1177 | - def test_10_validator(self): |
1178 | - self.assertFalse(validate_query('')) |
1179 | - self.assertTrue(validate_query('thisworks')) |
1180 | - self.assertTrue(validate_query('th1sw0rks')) |
1181 | - self.assertTrue(validate_query('12345')) |
1182 | - self.assertFalse(validate_query('this does not work')) |
1183 | - self.assertTrue(validate_query('this_does_work')) |
1184 | - self.assertFalse(validate_query("'")) # string consisting of a single quote mark |
1185 | - |
1186 | -if __name__ == '__main__': |
1187 | - unittest.main() |
1188 | |
1189 | === modified file 'src/sextant/test_parser.py' |
1190 | --- src/sextant/test_parser.py 2014-11-05 16:09:16 +0000 |
1191 | +++ src/sextant/test_parser.py 2014-11-25 10:17:59 +0000 |
1192 | @@ -11,20 +11,20 @@ |
1193 | def setUp(self): |
1194 | pass |
1195 | |
1196 | - def add_function(self, dct, name, typ): |
1197 | + def add_function(self, dct, name, typ, source): |
1198 | self.assertFalse(name in dct, "duplicate function added: {} into {}".format(name, dct.keys())) |
1199 | - dct[name] = typ |
1200 | + dct[name] = (typ, source) |
1201 | |
1202 | - def add_call(self, dct, caller, callee): |
1203 | - dct[caller].append(callee) |
1204 | + def add_call(self, dct, caller, callee, is_internal): |
1205 | + dct[caller].append((callee, is_internal)) |
1206 | |
1207 | def do_parse(self, path=DUMP_FILE, sections=['.text'], ignore_ptrs=False): |
1208 | functions = {} |
1209 | calls = defaultdict(list) |
1210 | |
1211 | # set the Parser to put output in local dictionaries |
1212 | - add_function = lambda n, t, s='unknown': self.add_function(functions, n, t) |
1213 | - add_call = lambda a, b: self.add_call(calls, a, b) |
1214 | + add_function = lambda n, t, s: self.add_function(functions, n, t, s) |
1215 | + add_call = lambda a, b, i: self.add_call(calls, a, b, i) |
1216 | |
1217 | p = parser.Parser(path, sections=sections, ignore_ptrs=ignore_ptrs, |
1218 | add_function=add_function, add_call=add_call) |
1219 | @@ -43,12 +43,16 @@ |
1220 | # ensure that the correct functions are listed with the correct types |
1221 | res, funcs, calls = self.do_parse() |
1222 | |
1223 | - for name, typ in zip(['normal', 'duplicates', 'wierd$name', 'printf', 'func_ptr_3'], |
1224 | - ['normal', 'normal', 'normal', 'stub', 'pointer']): |
1225 | + known = 'parser_test.c' |
1226 | + unknown = 'unknown' |
1227 | + |
1228 | + for name, typ, fle in zip(['normal', 'duplicates', 'wierd$name', 'printf', 'func_ptr_3'], |
1229 | + ['normal', 'normal', 'normal', 'stub', 'pointer'], |
1230 | + [known, known, known, unknown, unknown]): |
1231 | self.assertTrue(name in funcs, "'{}' not found in function dictionary".format(name)) |
1232 | - self.assertEquals(funcs[name], typ) |
1233 | + self.assertEquals(funcs[name][0], typ) |
1234 | + self.assertTrue(funcs[name][1].endswith(fle)) |
1235 | |
1236 | - self.assertFalse('__gmon_start__' in funcs, "don't see a function defined in .plt") |
1237 | |
1238 | def test_no_ptrs(self): |
1239 | # ensure that the ignore_ptrs flags is working |
1240 | @@ -61,17 +65,17 @@ |
1241 | def test_calls(self): |
1242 | res, funcs, calls = self.do_parse() |
1243 | |
1244 | - self.assertTrue('normal' in calls['main']) |
1245 | - self.assertTrue('duplicates' in calls['main']) |
1246 | + self.assertTrue(('normal', True) in calls['main']) |
1247 | + self.assertTrue(('duplicates', True) in calls['main']) |
1248 | |
1249 | normal_calls = sorted(['wierd$name', 'printf', 'func_ptr_3']) |
1250 | - self.assertEquals(sorted(calls['normal']), normal_calls) |
1251 | + self.assertEquals(sorted(zip(*calls['normal'])[0]), normal_calls) |
1252 | |
1253 | - self.assertEquals(calls['duplicates'].count('normal'), 2) |
1254 | - self.assertEquals(calls['duplicates'].count('printf'), 2, |
1255 | + self.assertEquals(calls['duplicates'].count(('normal', True)), 1) |
1256 | + self.assertEquals(calls['duplicates'].count(('printf', False)), 1, |
1257 | "expected 2 printf calls in {}".format(calls['duplicates'])) |
1258 | - self.assertTrue('func_ptr_4' in calls['duplicates']) |
1259 | - self.assertTrue('func_ptr_5' in calls['duplicates']) |
1260 | + self.assertTrue(('func_ptr_4', True) in calls['duplicates']) |
1261 | + self.assertTrue(('func_ptr_5', True) in calls['duplicates']) |
1262 | |
1263 | def test_sections(self): |
1264 | res, funcs, calls = self.do_parse(sections=['.plt', '.text']) |
1265 | |
1266 | === modified file 'src/sextant/test_resources/parser_test.dump' |
1267 | --- src/sextant/test_resources/parser_test.dump 2014-10-16 15:21:43 +0000 |
1268 | +++ src/sextant/test_resources/parser_test.dump 2014-11-25 10:17:59 +0000 |
1269 | @@ -20,20 +20,45 @@ |
1270 | 080483f0 <frame_dummy>: |
1271 | 804840f: call *%eax |
1272 | 0804841d <normal>: |
1273 | +/home/benhutc/filter-search/src/sextant/test_resources/parser_test.c:14 |
1274 | +/home/benhutc/filter-search/src/sextant/test_resources/parser_test.c:18 |
1275 | +/home/benhutc/filter-search/src/sextant/test_resources/parser_test.c:20 |
1276 | 8048430: call 8048458 <wierd$name> |
1277 | +/home/benhutc/filter-search/src/sextant/test_resources/parser_test.c:21 |
1278 | 8048443: call 80482f0 <printf@plt> |
1279 | +/home/benhutc/filter-search/src/sextant/test_resources/parser_test.c:22 |
1280 | 8048451: call *%eax |
1281 | +/home/benhutc/filter-search/src/sextant/test_resources/parser_test.c:24 |
1282 | +/home/benhutc/filter-search/src/sextant/test_resources/parser_test.c:25 |
1283 | 08048458 <wierd$name>: |
1284 | +/home/benhutc/filter-search/src/sextant/test_resources/parser_test.c:29 |
1285 | +/home/benhutc/filter-search/src/sextant/test_resources/parser_test.c:30 |
1286 | +/home/benhutc/filter-search/src/sextant/test_resources/parser_test.c:31 |
1287 | 08048460 <duplicates>: |
1288 | +/home/benhutc/filter-search/src/sextant/test_resources/parser_test.c:35 |
1289 | +/home/benhutc/filter-search/src/sextant/test_resources/parser_test.c:36 |
1290 | +/home/benhutc/filter-search/src/sextant/test_resources/parser_test.c:39 |
1291 | 804847b: call 80482f0 <printf@plt> |
1292 | +/home/benhutc/filter-search/src/sextant/test_resources/parser_test.c:40 |
1293 | 804848e: call 80482f0 <printf@plt> |
1294 | +/home/benhutc/filter-search/src/sextant/test_resources/parser_test.c:42 |
1295 | 8048499: call 804841d <normal> |
1296 | +/home/benhutc/filter-search/src/sextant/test_resources/parser_test.c:43 |
1297 | 80484a4: call 804841d <normal> |
1298 | +/home/benhutc/filter-search/src/sextant/test_resources/parser_test.c:45 |
1299 | 80484b2: call *%eax |
1300 | +/home/benhutc/filter-search/src/sextant/test_resources/parser_test.c:46 |
1301 | 80484bd: call *%eax |
1302 | +/home/benhutc/filter-search/src/sextant/test_resources/parser_test.c:48 |
1303 | +/home/benhutc/filter-search/src/sextant/test_resources/parser_test.c:49 |
1304 | 080484c4 <main>: |
1305 | +/home/benhutc/filter-search/src/sextant/test_resources/parser_test.c:53 |
1306 | +/home/benhutc/filter-search/src/sextant/test_resources/parser_test.c:54 |
1307 | 80484d4: call 804841d <normal> |
1308 | +/home/benhutc/filter-search/src/sextant/test_resources/parser_test.c:55 |
1309 | 80484e0: call 8048460 <duplicates> |
1310 | +/home/benhutc/filter-search/src/sextant/test_resources/parser_test.c:56 |
1311 | +/home/benhutc/filter-search/src/sextant/test_resources/parser_test.c:57 |
1312 | 080484f0 <__libc_csu_init>: |
1313 | 80484f6: call 8048350 <__x86.get_pc_thunk.bx> |
1314 | 804850e: call 80482b4 <_init> |
1315 | |
1316 | === modified file 'src/sextant/update_db.py' |
1317 | --- src/sextant/update_db.py 2014-11-13 14:01:55 +0000 |
1318 | +++ src/sextant/update_db.py 2014-11-25 10:17:59 +0000 |
1319 | @@ -9,9 +9,9 @@ |
1320 | |
1321 | __all__ = ("upload_program", "delete_program") |
1322 | |
1323 | -from .db_api import SextantConnection |
1324 | -from .sshmanager import SSHConnectionError |
1325 | -from .objdump_parser import Parser, run_objdump |
1326 | +from db_api import SextantConnection |
1327 | +from sshmanager import SSHConnectionError |
1328 | +from objdump_parser import Parser, run_objdump |
1329 | from os import path |
1330 | from time import time |
1331 | import subprocess |
1332 | |
1333 | === modified file 'src/sextant/web/server.py' |
1334 | --- src/sextant/web/server.py 2014-11-13 17:30:38 +0000 |
1335 | +++ src/sextant/web/server.py 2014-11-25 10:17:59 +0000 |
1336 | @@ -132,19 +132,19 @@ |
1337 | ), |
1338 | 'functions_calling': ( |
1339 | CONNECTION.get_all_functions_calling, |
1340 | - ('func1',) |
1341 | + ('func1', 'limit_internal', 'max_depth') |
1342 | ), |
1343 | 'functions_called_by': ( |
1344 | CONNECTION.get_all_functions_called, |
1345 | - ('func1',) |
1346 | + ('func1', 'limit_internal', 'max_depth') |
1347 | ), |
1348 | 'all_call_paths': ( |
1349 | CONNECTION.get_call_paths, |
1350 | - ('func1', 'func2') |
1351 | + ('func1', 'func2', 'limit_internal', 'max_depth') |
1352 | ), |
1353 | 'shortest_call_path': ( |
1354 | CONNECTION.get_shortest_path_between_functions, |
1355 | - ('func1', 'func2') |
1356 | + ('func1', 'func2', 'limit_internal', 'max_depth') |
1357 | ) |
1358 | } |
1359 | |
1360 | @@ -187,6 +187,8 @@ |
1361 | res_code = RESPONSE_CODE_BAD_REQUEST |
1362 | res_msg = "{}".format(e) |
1363 | print('\tfailed {}'.format(datetime.now())) |
1364 | + raise |
1365 | + |
1366 | |
1367 | if res_code is RESPONSE_CODE_OK: |
1368 | # we have received a response to our request |
1369 | @@ -237,7 +239,6 @@ |
1370 | max_funcs = AUTOCOMPLETE_NAMES_LIMIT + 1 |
1371 | programs = CONNECTION.programs_with_metadata() |
1372 | result = CONNECTION.get_function_names(program_name, search, max_funcs) |
1373 | - print(search, len(result)) |
1374 | return result if len(result) < max_funcs else set() |
1375 | |
1376 |