Merge lp:~ensoft-opensource/ensoft-sextant/auditing into lp:ensoft-sextant

[Patrick Stevens]

Add query `sextant audit`, and add metadata to Sextant programs (user id, username, date).

Add simple auditing of who is doing what with Sextant, by attaching to each program node a username (derived from the operating system), a user-id as returned by getuid, and a date-of-upload.
Add a new query, `sextant audit`, to make use of this.

[lp:~ensoft-opensource/ensoft-sextant/auditing]

11. By Patrick Stevens <email address hidden> on 2014-08-21

Add sextant audit command

12. By Patrick Stevens <email address hidden> on 2014-08-22

Remove an obsolete line from a docstring

13. By Patrick Stevens <email address hidden> on 2014-08-22

Fix too-early error handling when server couldn't be found in Audit, and give a message if Audit sees an empty database.

[James (jamesh-f)]

Couple of minor style new lines sorry :)
Also wit the neo4j connection ensure if you remove the nargs that you also remove the indexing

[lp:~ensoft-opensource/ensoft-sextant/auditing]

14. By Patrick Stevens <email address hidden> on 2014-08-22

Remove extraneous newline

[James (jamesh-f)]

Good Good

review: Approve

[lp:~ensoft-opensource/ensoft-sextant/auditing]

15. By Patrick Stevens <email address hidden> on 2014-08-26

Merge better-program-names, fix file permissions

[James (jamesh-f)]

All looks good to me

review: Approve

=== modified file 'doc/Program_upload_docs.mkd' (properties changed: +x to -x)
=== modified file 'doc/wiki/Reference' (properties changed: +x to -x)
=== modified file 'resources/sextant/web/interface.html' (properties changed: +x to -x)
=== modified file 'resources/sextant/web/queryjavascript.js' (properties changed: +x to -x)
=== modified file 'src/sextant/__main__.py' (properties changed: +x to -x)
--- src/sextant/__main__.py	2014-08-26 11:01:03 +0000
+++ src/sextant/__main__.py	2014-08-26 11:05:54 +0000
@@ -151,11 +151,30 @@
                             help='port on which to serve Sextant Web',
                             default=web_port)
 
+parsers['audit'] = subparsers.add_parser('audit', help='view usage of Sextant')
+
 for parser_key in parsers:
-    parsers[parser_key].add_argument('--remote-neo4j', metavar="URL", nargs=1,
+    parsers[parser_key].add_argument('--remote-neo4j', metavar="URL",
                                      help="URL of neo4j server", type=str,
                                      default=remote_neo4j)
 
+def _audit(args):
+    try:
+        audited = query.audit(args.remote_neo4j)
+    except requests.exceptions.ConnectionError as e:
+        logging.error('Connection error to server {}: {}'.format(args.remote_neo4j, e))
+
+    if not audited:
+        print('No programs on database at {}.'.format(args.remote_neo4j))
+    else:
+        for program in audited:
+            st = 'Program {} with {} functions uploaded by {} (id {}) on {}.'
+            print(st.format(program.program_name, program.number,
+                            program.uploader, program.uploader_id,
+                            program.date))
+    
+parsers['audit'].set_defaults(func=_audit)
+
 def _start_web(args):
     # Don't import at top level -- this makes twisted dependency semi-optional,
     # allowing non-web functionality to work with Python 3.
@@ -182,7 +201,8 @@
     try:
         update_db.upload_program(namespace.input_file[0],
                                  namespace.remote_neo4j,
-                                 alternative_name, not_object_file)
+                                 alternative_name,
+                                 not_object_file)
     except requests.exceptions.ConnectionError as e:
         logging.error('Connection error to server {}: {}'.format(namespace.remote_neo4j,
                                                                  e))

=== modified file 'src/sextant/db_api.py'
--- src/sextant/db_api.py	2014-08-19 09:20:45 +0000
+++ src/sextant/db_api.py	2014-08-26 11:05:54 +0000
@@ -10,6 +10,10 @@
 
 import re  # for validation of function/program names
 import logging
+from datetime import datetime
+import os
+import getpass
+from collections import namedtuple
 
 from neo4jrestclient.client import GraphDatabase
 import neo4jrestclient.client as client
@@ -45,12 +49,16 @@
 class AddToDatabase():
     """Updates the database, adding functions/calls to a given program"""
 
-    def __init__(self, program_name='', sextant_connection=None):
+    def __init__(self, program_name='', sextant_connection=None,
+                 uploader='', uploader_id='', date=None):
         """
         Object which can be used to add functions and calls to a new program
         :param program_name: the name of the new program to be created
           (must already be validated against Validator)
         :param sextant_connection: the SextantConnection to use for connections
+        :param uploader: string identifier of user who is uploading
+        :param uploader_id: string Unix user-id of logged-in user
+        :param date: string date of today
         """
         # program_name must be alphanumeric, to avoid injection attacks easily
         if not Validator.validate(program_name):
@@ -65,7 +73,11 @@
             # we'll locally use db for short
             db = self.parent_database_connection._db
 
-            parent_function = db.nodes.create(name=program_name, type='program')
+            parent_function = db.nodes.create(name=program_name,
+                                              type='program',
+                                              uploader=uploader,
+                                              uploader_id=uploader_id,
+                                              date=date)
             self._parent_id = parent_function.id
 
             self._new_tx = db.transaction(using_globals=False, for_query=True)
@@ -360,6 +372,10 @@
     It can be used to create/delete/query programs.
     """
 
+    ProgramWithMetadata = namedtuple('ProgramWithMetadata',
+                                     ['uploader', 'uploader_id',
+                                      'program_name', 'date', 'number'])
+
     def __init__(self, url):
         self.url = url
         self._db = GraphDatabase(url)
@@ -381,9 +397,14 @@
         if not Validator.validate(name_of_program):
             raise ValueError(
                 "{} is not a valid program name".format(name_of_program))
+        
+        uploader = getpass.getuser()
+        uploader_id = os.getuid()
 
         return AddToDatabase(sextant_connection=self,
-                             program_name=name_of_program)
+                             program_name=name_of_program,
+                             uploader=uploader, uploader_id=uploader_id,
+                             date=str(datetime.now()))
 
     def delete_program(self, name_of_program):
         """
@@ -435,6 +456,19 @@
 
         return set(result)
 
+    def programs_with_metadata(self):
+        """
+        Returns a set of namedtuples which represent the current database.
+        The namedtuples have .uploader, .uploader_id, .program_name, .date,
+        .number (of functions)
+        :return: set of namedtuples
+        """
+        q = ("MATCH (base) WHERE base.type = 'program' "
+             "MATCH (base)-[:subject]->(n)"
+             "RETURN base.uploader, base.uploader_id, base.name, base.date, count(n)")
+        result = self._db.query(q)
+        return {self.ProgramWithMetadata(*res) for res in result}
+
     def check_program_exists(self, program_name):
         """
         Execute query to check whether a program with the given name exists.

=== modified file 'src/sextant/export.py' (properties changed: +x to -x)
=== modified file 'src/sextant/query.py' (properties changed: +x to -x)
--- src/sextant/query.py	2014-08-26 11:01:03 +0000
+++ src/sextant/query.py	2014-08-26 11:05:54 +0000
@@ -13,6 +13,7 @@
 from . import db_api
 from .export import ProgramConverter
 
+
 def query(remote_neo4j, input_query, program_name=None, argument_1=None, argument_2=None, suppress_common=False):
 
     try:
@@ -94,5 +95,11 @@
     else:
         print('Nothing was returned from the query.')
 
+
+def audit(remote_neo4j):
+    db = db_api.SextantConnection(remote_neo4j)
+
+    return db.programs_with_metadata()
+
 if __name__ == '__main__':
     main()

=== modified file 'src/sextant/update_db.py'
--- src/sextant/update_db.py	2014-08-18 14:23:06 +0000
+++ src/sextant/update_db.py	2014-08-26 11:05:54 +0000
@@ -13,7 +13,9 @@
 import logging
 
 
-def upload_program(file_path, db_url, alternative_name=None, not_object_file=False):
+def upload_program(file_path, db_url, 
+                   alternative_name=None,
+                   not_object_file=False):
     """
     Uploads a program to the remote database.
     Raises requests.exceptions.ConnectionError if the server didn't exist.
@@ -40,15 +42,19 @@
                              "please delete the previous one with the same name"
                              " and retry, or rename the input file.")
 
-    parsed_objects = get_parsed_objects(filepath=file_path, sections_to_view=['.text'],
-                                        not_object_file=not_object_file, ignore_function_pointers=False)
+    parsed_objects = get_parsed_objects(filepath=file_path,
+                                        sections_to_view=['.text'],
+                                        not_object_file=not_object_file,
+                                        ignore_function_pointers=False)
 
     logging.info('Objdump has parsed!')
 
     if alternative_name is None:
-        program_representation = connection.new_program(Validator.sanitise(file_path))
+        program_representation = connection.new_program(
+                                     Validator.sanitise(file_path))
     else:
-        program_representation = connection.new_program(Validator.sanitise(alternative_name))
+        program_representation = connection.new_program(
+                                     Validator.sanitise(alternative_name))
 
     for obj in parsed_objects:
         for called in obj.what_do_i_call:

=== modified file 'src/sextant/web/server.py' (properties changed: +x to -x)