Merge lp:~thomir-deactivatedaccount/adt-cloud-service/trunk-fix-logging into lp:adt-cloud-service

[Thomi Richards]

Sort out logging.

Sort out logging for the a-c-w:

 - we keep a separate application log.
  - we try and write to 'logs/app.log'
  - we fall back to stderr if the logs directory does not exist.
 - we configure the logstash handler if our config file contains the 'logstash' section.
 - we log every request that contains missing parameters, and every time we enqueue something to rabbit.

[Joe Talbott (joetalbott)]

One question in-line.

[Paul Larson (pwlars)]

Neat, I didn't know you could send directly to logstash like that. Looks good.

review: Approve

[Celso Providelo (cprov)]

review: Approve

=== modified file 'README.rst'
--- README.rst	2015-03-16 20:56:36 +0000
+++ README.rst	2015-03-19 22:03:02 +0000
@@ -30,6 +30,14 @@
   [image_mapper]
   url = http://127.0.0.1:1234
 
+Optionally, add a `[logstash]` section, which will enable the logstash logging handler::
+
+  [logstash]
+  host = localhost
+  port = 5959
+  version = 1
+
+
 Running::
 
   $ gunicorn adt_cloud_service:app

=== modified file 'adt-service.conf'
--- adt-service.conf	2015-03-16 20:56:36 +0000
+++ adt-service.conf	2015-03-19 22:03:02 +0000
@@ -1,6 +1,6 @@
 # `adt-cloud-worker` configuration file.
 [image_mapper]
-url = http://127.0.0.1:1234
+url = http://127.0.0.1:5000
 
 [amqp]
 uris = amqp://guest:guest@localhost:5672//
@@ -8,9 +8,7 @@
 [swift]
 public_url = http://some.swift.server.com
 
-[nova]
-os_username = foo
-os_tenant_name = foo_project
-os_password = xxx
-os_auth_url = http://172.20.161.138:5000/v2.0/
-os_region_name = bot_prototype
+[logstash]
+host = localhost
+port = 5959
+version = 1

=== modified file 'adt_cloud_service/__init__.py'
--- adt_cloud_service/__init__.py	2015-03-18 18:59:21 +0000
+++ adt_cloud_service/__init__.py	2015-03-19 22:03:02 +0000
@@ -16,8 +16,11 @@
 #
 
 import configparser
+import logging
+import os.path
 
 import flask
+import logstash
 
 from adt_cloud_service import (
     errors,
@@ -33,6 +36,8 @@
         config = configparser.ConfigParser()
         config.read('adt-service.conf')
 
+    configure_logging(config)
+
     app = flask.Flask('adt_cloud_service')
 
     app.add_url_rule(
@@ -70,4 +75,33 @@
     return app
 
 
+def configure_logging(config):
+    root_logger = logging.getLogger()
+    root_logger.setLevel(logging.INFO)
+
+    log_path = os.path.abspath(os.path.join(__file__, '../../logs/app.log'))
+    log_dir = os.path.dirname(log_path)
+    handler = None
+    if os.path.exists(log_dir):
+        handler = logging.FileHandler(log_path)
+    else:
+        print("'logs' directory '{}' does not exist, using stderr for app log.".format(log_dir))
+        handler = logging.StreamHandler()
+    handler.setFormatter(
+        logging.Formatter(
+            '%(asctime)s  %(name)s %(levelname)s: %(message)s'
+        )
+    )
+    root_logger.addHandler(handler)
+
+    if 'logstash' in config:
+        root_logger.addHandler(
+            logstash.LogstashHandler(
+                config['logstash']['host'],
+                int(config['logstash']['port']),
+                int(config['logstash']['version'])
+            )
+        )
+
+
 app = create_flask_app()

=== modified file 'adt_cloud_service/queue.py'
--- adt_cloud_service/queue.py	2015-03-17 02:47:19 +0000
+++ adt_cloud_service/queue.py	2015-03-19 22:03:02 +0000
@@ -21,6 +21,9 @@
 from kombu.pools import producers
 from kombu.common import uuid
 
+import logging
+logger = logging.getLogger(__name__)
+
 
 class RabbitQueuer(object):
     """A class that can send things to the rabbit queue. An instance of this
@@ -78,4 +81,5 @@
                 exchange=self.adt_exchange,
                 routing_key=routing_key,
                 declare=declare)
+        logger.info("Submitted payload %r", payload_copy, extra=payload_copy)
         return payload_copy['request_id']

=== modified file 'adt_cloud_service/v1.py'
--- adt_cloud_service/v1.py	2015-03-18 20:20:47 +0000
+++ adt_cloud_service/v1.py	2015-03-19 22:03:02 +0000
@@ -17,12 +17,16 @@
 
 """Version 1 restful service web code."""
 
+import logging
 import urllib.parse
 
 from flask import current_app, request, url_for, jsonify
 
 from adt_cloud_service.errors import MissingRequestParameters
 
+logger = logging.getLogger(__name__)
+
+
 def test_new():
     required_keys = {
         'package-name',
@@ -32,6 +36,7 @@
     }
     missing = required_keys.difference(set(request.form.keys()))
     if missing:
+        logger.error("Got request %r with missing keys %r", dict(request.form), missing)
         raise MissingRequestParameters(missing)
 
     rabbit_request = dict()

=== modified file 'requirements.txt'
--- requirements.txt	2015-03-12 18:56:59 +0000
+++ requirements.txt	2015-03-19 22:03:02 +0000
@@ -1,3 +1,4 @@
 flask==0.10.1
 gunicorn==19.3.0
 kombu==3.0.24
+python-logstash==0.4.2