Merge lp:~pconnell/ensoft-sextant/entrails into lp:~ensoft-opensource/ensoft-sextant/trunk

[Phil Connell]

Integrate with entrails to enable runtime tracing of Python programs.

Upshot: it's now possible to get callgraphs from Python programs into Sextant.

Specifically, a new context manager is added, sextant.trace(). Any calls inside the with block are traced.

There are also a few minor bugfixes (mostly related to Python 3 support). Commit messages have some details.

[Patrick Stevens (patrickas)]

I don't see anything particularly wrong with this, although I am not the world's greatest Python programmer!

[lp:~pconnell/ensoft-sextant/entrails]

180. By Phil Connell <email address hidden> on 2014-08-15

Using logging, not print in sextant.__main__

Setup of logging is a bit hacked in, something to clean up later.

[Phil Connell (pconnell)]

Suggestion to use logging rather than print is definitely valid. New request coming...

Unmerged revisions

=== modified file 'src/sextant/__init__.py'
--- src/sextant/__init__.py	2014-08-11 15:14:05 +0000
+++ src/sextant/__init__.py	2014-08-15 10:49:39 +0000
@@ -5,9 +5,17 @@
 # -----------------------------------------
 """Program call graph recording and query framework."""
 
+from . import errors
+from . import pyinput
+
 __all__ = (
     "SextantConnection",
+) + (
+    errors.__all__ +
+    pyinput.__all__
 )
 
-from .update_db import SextantConnection
+from .db_api import SextantConnection
+from .errors import *
+from .pyinput import *
 

=== modified file 'src/sextant/__main__.py'
--- src/sextant/__main__.py	2014-08-14 14:41:19 +0000
+++ src/sextant/__main__.py	2014-08-15 10:49:39 +0000
@@ -5,13 +5,42 @@
 # -----------------------------------------
 #invokes Sextant and argparse
 
+from __future__ import absolute_import, print_function
+
 import argparse
-import ConfigParser, os, sys
+try:
+    import ConfigParser
+except ImportError:
+    import configparser as ConfigParser
+import logging
+import logging.config
+import os
+import sys
 
 from . import update_db
 from . import query
 from . import db_api
 
+
+# @@@ Logging level should be configurable (with some structure to setting up
+# logging).
+logging.config.dictConfig({
+    "version": 1,
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+            "level": logging.INFO,
+            "stream": "ext://sys.stderr",
+        },
+    },
+    "root": {
+        "level": logging.DEBUG,
+        "handlers": ["console"],
+    },
+})
+log = logging.getLogger()
+
+
 def get_config_file():
     # get the config file option for neo4j server location and web port number
     _ROOT = os.path.abspath(os.path.dirname(__file__))
@@ -23,23 +52,18 @@
     example_config = get_data('../etc', 'sextant.conf')
 
     try:
-        conffile = (p for p in (home_config, env_config, example_config) if os.path.exists(p)).next()
+        conffile = next(p
+                        for p in (home_config, env_config, example_config)
+                        if os.path.exists(p))
     except StopIteration:
         #No config files accessable
-        if os.environ.has_key("SEXTANT_CONFIG"):
+        if "SEXTANT_CONFIG" in os.environ:
             #SEXTANT_CONFIG environment variable is set
-            print("SEXTANT_CONFIG currently {}, is set incorrectly.".format(env_config))
-        print("Sextant requires a configuration file.\n")
-        sys.exit(1)
-
-    if conffile is example_config and os.environ.has_key("SEXTANT_CONFIG"):
-        #example config file is available but the user has set the
-        # environment variable, therefore we assume the user actually
-        # wants to use a different config file.
-        print("SEXTANT_CONFIG currently {}, is set incorrectly.".format(env_config))
-        sys.exit(1)
-
-    print("Sextant using config file{}".format(conffile))
+            log.error("SEXTANT_CONFIG file %r doesn't exist.", env_config)
+        log.error("Sextant requires a configuration file.")
+        sys.exit(1)
+
+    log.info("Sextant is using config file %s", conffile)
     return conffile
 
 conffile = get_config_file()
@@ -129,8 +153,8 @@
 def _start_web(args):
     # Don't import at top level -- this makes twisted dependency semi-optional,
     # allowing non-web functionality to work with Python 3.
-    from web import server
-    print("Serving site on port {}".format(args.port))
+    from .web import server
+    log.info("Serving site on port %s", args.port)
     server.serve_site(input_database_url=args.remote_neo4j, port=args.port)
 
 parsers['web'].set_defaults(func=_start_web)

=== modified file 'src/sextant/db_api.py'
--- src/sextant/db_api.py	2014-08-14 13:53:59 +0000
+++ src/sextant/db_api.py	2014-08-15 10:49:39 +0000
@@ -375,14 +375,15 @@
         The name specified must pass Validator.validate()ion; this is a measure
           to prevent Cypher injection attacks.
         :param name_of_program: string program name
-        :return: False if unsuccessful, or an AddToDatabase if successful
+        :return: AddToDatabase instance if successful
         """
 
         if not Validator.validate(name_of_program):
-            return False
+            raise ValueError(
+                "{} is not a valid program name".format(name_of_program))
 
         return AddToDatabase(sextant_connection=self,
-                             program_name=name_of_program) or False
+                             program_name=name_of_program)
 
     def delete_program(self, name_of_program):
         """

=== added file 'src/sextant/errors.py'
--- src/sextant/errors.py	1970-01-01 00:00:00 +0000
+++ src/sextant/errors.py	2014-08-15 10:49:39 +0000
@@ -0,0 +1,20 @@
+# -----------------------------------------------------------------------------
+# errors.py -- Sextant error definitions
+#
+# August 2014, Phil Connell
+#
+# Copyright 2014, Ensoft Ltd.
+# -----------------------------------------------------------------------------
+
+from __future__ import absolute_import, print_function
+
+__all__ = (
+    "MissingDependencyError",
+)
+
+
+class MissingDependencyError(Exception):
+    """
+    Raised if trying an operation for which an optional dependency is missing.
+    """
+

=== added file 'src/sextant/pyinput.py'
--- src/sextant/pyinput.py	1970-01-01 00:00:00 +0000
+++ src/sextant/pyinput.py	2014-08-15 10:49:39 +0000
@@ -0,0 +1,180 @@
+# -----------------------------------------------------------------------------
+# pyinput.py -- Input information from Python programs.
+#
+# August 2014, Phil Connell
+#
+# Copyright 2014, Ensoft Ltd.
+# -----------------------------------------------------------------------------
+
+from __future__ import absolute_import, print_function
+
+__all__ = (
+    "trace",
+)
+
+
+import contextlib
+import sys
+
+from . import errors
+from . import db_api
+
+
+# Optional, should be checked at API entrypoints requiring entrails (and
+# yes, the handling is a bit fugly).
+try:
+    import entrails
+except ImportError:
+    _entrails_available = False
+    class entrails:
+        EntrailsOutput = object
+else:
+    _entrails_available = True
+
+
+class _SextantOutput(entrails.EntrailsOutput):
+    """Record calls traced by entrails in a sextant database."""
+
+    # Internal attributes:
+    #
+    # _conn:
+    #   Sextant connection.
+    # _fns:
+    #   Stack of function names (implemented as a list), reflecting the current
+    #   call stack, based on enter, exception and exit events.
+    # _prog:
+    #   Sextant program representation.
+    _conn = None
+    _fns = None
+    _prog = None
+
+    def __init__(self, conn, program_name):
+        """
+        Initialise this output.
+
+        conn:
+            Connection to the Sextant database.
+        program_name:
+            String used to refer to the traced program in sextant.
+
+        """
+        self._conn = conn
+        self._fns = []
+        self._prog = self._conn.new_program(program_name)
+        self._tracer = self._trace()
+        next(self._tracer)
+
+    def _add_frame(self, event):
+        """Add a function call to the internal stack."""
+        name = event.qualname()
+        self._fns.append(name)
+        self._prog.add_function(name)
+
+        try:
+            prev_name = self._fns[-2]
+        except IndexError:
+            pass
+        else:
+            self._prog.add_function_call(prev_name, name)
+
+    def _remove_frame(self, event):
+        """Remove a function call from the internal stack."""
+        assert event.qualname() == self._fns[-1], \
+                "Unexpected event for {}".format(event.qualname())
+        self._fns.pop()
+
+    def _handle_simple_event(self, what, event):
+        """Handle a single trace event, not needing recursive processing."""
+        handled = True
+
+        if what == "enter":
+            self._add_frame(event)
+        elif what == "exit":
+            self._remove_frame(event)
+        else:
+            handled = False
+
+        return handled
+
+    def _trace(self):
+        """Coroutine that processes trace events it's sent."""
+        while True:
+            what, event = yield
+
+            handled = self._handle_simple_event(what, event)
+            if not handled:
+                if what == "exception":
+                    # An exception doesn't necessarily mean the current stack
+                    # frame is exiting. Need to check whether the next event is
+                    # an exception in a different stack frame, implying that
+                    # the exception is propagating up the stack.
+                    while True:
+                        prev_event = event
+                        prev_name = event.qualname()
+                        what, event = yield
+                        if event == "exception":
+                            if event.qualname() != prev_name:
+                                self._remove_frame(prev_event)
+                        else:
+                            handled = self._handle_simple_event(what, event)
+                            assert handled
+                            break
+
+                else:
+                    raise NotImplementedError
+
+    def close(self):
+        self._prog.commit()
+
+    def enter(self, event):
+        self._tracer.send(("enter", event))
+
+    def exception(self, event):
+        self._tracer.send(("exception", event))
+
+    def exit(self, event):
+        self._tracer.send(("exit", event))
+
+
+# @@@ config parsing shouldn't be done in __main__ (we want to get the neo4j
+# url from there...)
[e-mail]
+def trace(conn, program_name=None, filters=None):
+    """
+    Context manager that records function calls in its context block.
+
+    e.g. given this code:
+
+        with sextant.trace("http://localhost:7474"):
+            foo()
+            bar()
+
+    The calls to foo() and bar() (and their callees, at any depth) will be
+    recorded in the sextant database.
+
+    conn:
+        Instance of SextantConnection that will be used to record calls.
+    program_name:
+        String used to refer to the traced program in sextant. Defaults to
+        sys.argv[0].
+    filters:
+        Optional iterable of entrails filters to apply.
+
+    """
+    if not _entrails_available:
+        raise errors.MissingDependencyError(
+            "Entrails is required to trace execution")
+
+    if program_name is None:
+        program_name = sys.argv[0]
+
+    tracer = entrails.Entrails(filters=filters)
+    tracer.add_output(_SextantOutput(conn, program_name))
+
+    tracer.start_trace()
+    try:
+        yield
+    finally:
+        # Flush traced data.
+        tracer.end_trace()
+

=== modified file 'src/sextant/web/server.py'
--- src/sextant/web/server.py	2014-08-14 09:03:17 +0000
+++ src/sextant/web/server.py	2014-08-15 10:49:39 +0000
@@ -301,7 +301,10 @@
 
     global database_url
     database_url = input_database_url
-    root = File(os.path.dirname(os.path.abspath(__file__))+'/../../resources/web/')  # serve static directory at root
+    # serve static directory at root
+    root = File(os.path.join(
+        os.path.dirname(os.path.abspath(__file__)),
+        "..", "..", "..", "resources", "web"))
 
     # serve a dynamic Echoer webpage at /echoer.html
     root.putChild("echoer.html", Echoer())