Merge lp:~theiw/txstatsd/txstatsd-logging-processor into lp:txstatsd

[Ian Wilkinson]

Support recording metric samples to a log file.

Now includes the GaugeMetricReporter.

Support recording metric samples to a log file.

Now includes the GaugeMetricReporter.

[Lucio Torre (lucio.torre)]

review: Approve

[Ubuntu One Server Tarmac Bot (ubuntuone-server-tarmac)]

The attempt to merge lp:~theiw/txstatsd/txstatsd-logging-processor into lp:txstatsd failed. Below is the output from the failed tests.

txstatsd.tests.metrics.test_histogrammetric
  TestHistogramReporterMetric
    test_histogram_of_numbers_1_through_10000 ... [OK]
    test_histogram_with_zero_recorded_values ... [OK]
txstatsd.tests.metrics.test_timermetric
  TestBlankTimerMetric
    test_count ... [OK]
    test_fifteen_minute_rate ... [OK]
    test_five_minute_rate ... [OK]
    test_max ... [OK]
    test_mean ... [OK]
    test_mean_rate ... [OK]
    test_min ... [OK]
    test_no_values ... [OK]
    test_one_minute_rate ... [OK]
    test_percentiles ... [OK]
    test_std_dev ... [OK]
  TestTimingSeriesEvents
    test_count ... [OK]
    test_max ... [OK]
    test_mean ... [OK]
    test_min ... [OK]
    test_percentiles ... [OK]
    test_std_dev ... [OK]
    test_values ... [OK]
txstatsd.tests.stats.test_ewma
  TestEwmaFifteenMinute
    test_eight_minutes ... [OK]
    test_eleven_minutes ... [OK]
    test_fifteen_minutes ... [OK]
    test_first_tick ... [OK]
    test_five_minutes ... [OK]
    test_four_minutes ... [OK]
    test_fourteen_minutes ... [OK]
    test_nine_minutes ... [OK]
    test_one_minute ... [OK]
    test_seven_minutes ... [OK]
    test_six_minutes ... [OK]
    test_ten_minutes ... [OK]
    test_thirteen_minutes ... [OK]
    test_three_minutes ... [OK]
    test_twelve_minutes ... [OK]
    test_two_minutes ... ...

[Ian Wilkinson (theiw)]

Tests changed to be python 2.6 friendly.

review: Approve

[Ubuntu One Server Tarmac Bot (ubuntuone-server-tarmac)]

There are additional revisions which have not been approved in review. Please seek review and approval of these new revisions.

=== added file 'requirements.txt'
--- requirements.txt	1970-01-01 00:00:00 +0000
+++ requirements.txt	2011-10-05 16:03:23 +0000
@@ -0,0 +1,5 @@
+Twisted==11.0.0
+mocker==1.1
+psutil==0.3.0
+wsgiref==0.1.2
+zope.interface==3.8.0

=== modified file 'txstatsd/metrics/gaugemetric.py'
--- txstatsd/metrics/gaugemetric.py	2011-07-03 11:49:05 +0000
+++ txstatsd/metrics/gaugemetric.py	2011-10-05 16:03:23 +0000
@@ -1,4 +1,6 @@
 
+from string import Template
+
 from txstatsd.metrics.metric import Metric
 
 
@@ -20,3 +22,33 @@
     def mark(self, value):
         """Report the C{value} for this gauge."""
         self.send("%s|g" % value)
+
+
+class GaugeMetricReporter(object):
+    """A gauge metric is an instantaneous reading of a particular value."""
+
+    MESSAGE = (
+        "$prefix%(key)s.value %(value)s %(timestamp)s\n")
+
+    def __init__(self, name, prefix=""):
+        """Construct a metric we expect to be periodically updated.
+
+        @param name: Indicates what is being instrumented.
+        """
+        self.name = name
+
+        if prefix:
+            prefix += '.'
+        self.message = Template(GaugeMetricReporter.MESSAGE).substitute(
+            prefix=prefix)
+
+        self.value = 0
+
+    def mark(self, value):
+        self.value = value
+
+    def report(self, timestamp):
+        return self.message % {
+            "key": self.name,
+            "value": self.value,
+            "timestamp": timestamp}

=== modified file 'txstatsd/server/configurableprocessor.py'
--- txstatsd/server/configurableprocessor.py	2011-09-14 12:01:10 +0000
+++ txstatsd/server/configurableprocessor.py	2011-10-05 16:03:23 +0000
@@ -1,9 +1,8 @@
 
-from string import Template
-
 import time
 
 from txstatsd.metrics.countermetric import CounterMetricReporter
+from txstatsd.metrics.gaugemetric import GaugeMetricReporter
 from txstatsd.metrics.metermetric import MeterMetricReporter
 from txstatsd.metrics.timermetric import TimerMetricReporter
 from txstatsd.server.processor import MessageProcessor
@@ -17,26 +16,18 @@
     Currently, this extends to:
     - Allow a prefix to be added to the composed messages sent
       to the Graphite server.
+    - Report an instantaneous reading of a particular value.
     - Report an incrementing and decrementing counter metric.
     - Report a timer metric which aggregates timing durations and provides
       duration statistics, plus throughput statistics.
     """
 
-    #TODO Provide a GaugeMetricReporter (in the manner of the other
-    # Coda Hale metrics).
-    GAUGE_METRIC_MESSAGE = (
-        "$prefix%(key)s.value %(value)s %(timestamp)s\n")
-
     def __init__(self, time_function=time.time, message_prefix=""):
-        super(ConfigurableMessageProcessor, self).__init__(time_function)
+        super(ConfigurableMessageProcessor, self).__init__(
+            time_function=time_function)
 
         self.message_prefix = message_prefix
-        if message_prefix:
-            message_prefix += '.'
-
-        message = Template(ConfigurableMessageProcessor.GAUGE_METRIC_MESSAGE)
-        self.gauge_metric_message = message.substitute(
-            prefix=message_prefix)
+        self.gauge_metrics = {}
 
     def compose_timer_metric(self, key, duration):
         if not key in self.timer_metrics:
@@ -58,6 +49,12 @@
             self.counter_metrics[key] = metric
         self.counter_metrics[key].mark(value)
 
+    def compose_gauge_metric(self, key, value):
+        if not key in self.gauge_metrics:
+            metric = GaugeMetricReporter(key, prefix=self.message_prefix)
+            self.gauge_metrics[key] = metric
+        self.gauge_metrics[key].mark(value)
+
     def compose_meter_metric(self, key, value):
         if not key in self.meter_metrics:
             metric = MeterMetricReporter(key, self.time_function,
@@ -75,6 +72,16 @@
 
         return (metrics, events)
 
+    def flush_gauge_metrics(self, timestamp):
+        metrics = []
+        events = 0
+        for metric in self.gauge_metrics.itervalues():
+            message = metric.report(timestamp)
+            metrics.append(message)
+            events += 1
+
+        return (metrics, events)
+
     def flush_timer_metrics(self, percent, timestamp):
         metrics = []
         events = 0

=== added file 'txstatsd/server/loggingprocessor.py'
--- txstatsd/server/loggingprocessor.py	1970-01-01 00:00:00 +0000
+++ txstatsd/server/loggingprocessor.py	2011-10-05 16:03:23 +0000
@@ -0,0 +1,37 @@
+
+import time
+
+from txstatsd.server.configurableprocessor import ConfigurableMessageProcessor
+
+
+class LoggingMessageProcessor(ConfigurableMessageProcessor):
+    """
+    This specialised C{MessageProcessor} logs the received metrics
+    using the supplied logger (which should have a callable C{info}
+    attribute.)
+    """
+
+    def __init__(self, logger, time_function=time.time, message_prefix=""):
+        super(LoggingMessageProcessor, self).__init__(
+            time_function=time_function, message_prefix=message_prefix)
+
+        logger_info = getattr(logger, 'info', None)
+        if logger_info is None or not callable(logger_info):
+            raise TypeError()
+        self.logger = logger
+
+    def flush(self):
+        """Log all received metric samples to the supplied log file."""
+        timestamp = int(self.time_function())
+
+        def log_metrics(metrics):
+            for metric in metrics.itervalues():
+                report = metric.report(timestamp)
+                for measurement in report.splitlines():
+                    self.logger.info(measurement)
+
+        log_metrics(self.counter_metrics)
+        log_metrics(self.gauge_metrics)
+        log_metrics(self.meter_metrics)
+        log_metrics(self.timer_metrics)
+

=== modified file 'txstatsd/server/processor.py'
--- txstatsd/server/processor.py	2011-09-14 12:01:10 +0000
+++ txstatsd/server/processor.py	2011-10-05 16:03:23 +0000
@@ -132,11 +132,16 @@
             return self.fail(message)
 
         try:
-            metric = [float(values[0]), key]
-            self.gauge_metrics.append(metric)
+            value = float(values[0])
         except (TypeError, ValueError):
             self.fail(message)
 
+        self.compose_gauge_metric(key, value)
+
+    def compose_gauge_metric(self, key, value):
+        metric = [value, key]
+        self.gauge_metrics.append(metric)
+
     def process_meter_metric(self, key, composite, message):
         values = composite.split(":")
         if not len(values) == 1:

=== modified file 'txstatsd/service.py'
--- txstatsd/service.py	2011-09-29 08:50:43 +0000
+++ txstatsd/service.py	2011-10-05 16:03:23 +0000
@@ -34,11 +34,7 @@
 
 
 class OptionsGlue(usage.Options):
-    """Extends usage.Options to also read parameters from a config file.
-
-    This is made far more complicated than should be necessary, as
-    usage.Options does not appear to have been designed for extension.
-    """
+    """Extends usage.Options to also read parameters from a config file."""
 
     optParameters = [
         ["config", "c", None, "Config file to use."]

=== added file 'txstatsd/tests/test_loggingprocessor.py'
--- txstatsd/tests/test_loggingprocessor.py	1970-01-01 00:00:00 +0000
+++ txstatsd/tests/test_loggingprocessor.py	2011-10-05 16:03:23 +0000
@@ -0,0 +1,45 @@
+
+from unittest import TestCase
+
+from txstatsd.server.loggingprocessor import LoggingMessageProcessor
+
+
+class FakeMeterMetric(object):
+    def report(self, *args):
+        return 'Sample report'
+
+
+class TestLoggingMessageProcessor(TestCase):
+
+    def test_logger_with_no_info(self):
+        def invoker():
+            logger = 'logger'
+            LoggingMessageProcessor(logger)
+
+        self.assertRaises(TypeError, invoker)
+
+    def test_logger_with_non_callable_info(self):
+        def invoker():
+            class Logger(object):
+                def __init__(self):
+                    self.info = 'logger'
+
+            logger = Logger()
+            LoggingMessageProcessor(logger)
+
+        self.assertRaises(TypeError, invoker)
+
+    def test_logger(self):
+        class Logger(object):
+            def __init__(self):
+                self.log = ''
+
+            def info(self, measurement):
+                self.log += measurement
+
+        logger = Logger()
+        processor = LoggingMessageProcessor(logger)
+        metric = FakeMeterMetric()
+        processor.meter_metrics['test'] = metric
+        processor.flush()
+        self.assertEqual(metric.report(), logger.log)

=== modified file 'txstatsd/version.py'
--- txstatsd/version.py	2011-09-29 08:50:43 +0000
+++ txstatsd/version.py	2011-10-05 16:03:23 +0000
@@ -1,1 +1,1 @@
-txstatsd = "0.5.1"
+txstatsd = "0.6.0"