Merge lp:~theiw/txstatsd/txstatsd-metermetric into lp:txstatsd
- txstatsd-metermetric
- Merge into trunk
Proposed by
Ian Wilkinson
Status: | Merged |
---|---|
Approved by: | Sidnei da Silva |
Approved revision: | 25 |
Merged at revision: | 25 |
Proposed branch: | lp:~theiw/txstatsd/txstatsd-metermetric |
Merge into: | lp:txstatsd |
Diff against target: |
756 lines (+627/-1) 10 files modified
README (+3/-0) txstatsd/metrics/metermetric.py (+104/-0) txstatsd/metrics/metrics.py (+11/-0) txstatsd/server/processor.py (+39/-0) txstatsd/service.py (+6/-0) txstatsd/stats/ewma.py (+70/-0) txstatsd/tests/stats/test_ewma.py (+315/-0) txstatsd/tests/test_metrics.py (+6/-0) txstatsd/tests/test_processor.py (+72/-0) txstatsd/version.py (+1/-1) |
To merge this branch: | bzr merge lp:~theiw/txstatsd/txstatsd-metermetric |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Sidnei da Silva | Approve | ||
Lucio Torre (community) | Approve | ||
Review via email: mp+73032@code.launchpad.net |
Commit message
Description of the change
Introduce support for Coda Hale's meter metric.
To post a comment you must log in.
Revision history for this message
Lucio Torre (lucio.torre) : | # |
review:
Approve
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'README' | |||
2 | --- README 2011-06-20 19:11:22 +0000 | |||
3 | +++ README 2011-08-26 12:28:49 +0000 | |||
4 | @@ -11,6 +11,9 @@ | |||
5 | 11 | * The txstatsd python package, containing a server and a client implementation | 11 | * The txstatsd python package, containing a server and a client implementation |
6 | 12 | for the statsd protocol. | 12 | for the statsd protocol. |
7 | 13 | 13 | ||
8 | 14 | * The metrics support borrows from Coda Hale's Metrics project | ||
9 | 15 | https://github.com/codahale/metrics. | ||
10 | 16 | |||
11 | 14 | License | 17 | License |
12 | 15 | ------- | 18 | ------- |
13 | 16 | 19 | ||
14 | 17 | 20 | ||
15 | === added file 'txstatsd/metrics/metermetric.py' | |||
16 | --- txstatsd/metrics/metermetric.py 1970-01-01 00:00:00 +0000 | |||
17 | +++ txstatsd/metrics/metermetric.py 2011-08-26 12:28:49 +0000 | |||
18 | @@ -0,0 +1,104 @@ | |||
19 | 1 | |||
20 | 2 | import time | ||
21 | 3 | |||
22 | 4 | from txstatsd.metrics.metric import Metric | ||
23 | 5 | from txstatsd.stats.ewma import Ewma | ||
24 | 6 | |||
25 | 7 | |||
26 | 8 | class MeterMetric(Metric): | ||
27 | 9 | """ | ||
28 | 10 | A meter metric which measures mean throughput and one-, five-, and | ||
29 | 11 | fifteen-minute exponentially-weighted moving average throughputs. | ||
30 | 12 | |||
31 | 13 | See: | ||
32 | 14 | - U{EMA | ||
33 | 15 | <http://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average>} | ||
34 | 16 | """ | ||
35 | 17 | |||
36 | 18 | def __init__(self, connection, name, sample_rate=1): | ||
37 | 19 | """Construct a metric that reports samples to the supplied | ||
38 | 20 | C{connection}. | ||
39 | 21 | |||
40 | 22 | @param connection: The connection endpoint representing | ||
41 | 23 | the StatsD server. | ||
42 | 24 | @param name: Indicates what is being instrumented. | ||
43 | 25 | @param sample_rate: Restrict the number of samples sent | ||
44 | 26 | to the StatsD server based on the supplied C{sample_rate}. | ||
45 | 27 | """ | ||
46 | 28 | Metric.__init__(self, connection, name, sample_rate=sample_rate) | ||
47 | 29 | |||
48 | 30 | def mark(self, value=1): | ||
49 | 31 | """Mark the occurrence of a given number (C{value}) of events.""" | ||
50 | 32 | self.send("%s|m" % value) | ||
51 | 33 | |||
52 | 34 | |||
53 | 35 | class MeterMetricReporter(object): | ||
54 | 36 | """ | ||
55 | 37 | A meter metric which measures mean throughput and one-, five-, and | ||
56 | 38 | fifteen-minute exponentially-weighted moving average throughputs. | ||
57 | 39 | |||
58 | 40 | See: | ||
59 | 41 | - U{EMA | ||
60 | 42 | <http://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average>} | ||
61 | 43 | """ | ||
62 | 44 | |||
63 | 45 | MESSAGE = ( | ||
64 | 46 | "stats.meter.%(key)s.count %(count)s %(timestamp)s\n" | ||
65 | 47 | "stats.meter.%(key)s.mean_rate %(mean_rate)s %(timestamp)s\n" | ||
66 | 48 | "stats.meter.%(key)s.1min_rate %(rate_1min)s %(timestamp)s\n" | ||
67 | 49 | "stats.meter.%(key)s.5min_rate %(rate_5min)s %(timestamp)s\n" | ||
68 | 50 | "stats.meter.%(key)s.15min_rate %(rate_15min)s %(timestamp)s\n") | ||
69 | 51 | |||
70 | 52 | def __init__(self, name, wall_time_func=time.time): | ||
71 | 53 | """Construct a metric we expect to be periodically updated. | ||
72 | 54 | |||
73 | 55 | @param name: Indicates what is being instrumented. | ||
74 | 56 | @param wall_time_func: Function for obtaining wall time. | ||
75 | 57 | """ | ||
76 | 58 | self.name = name | ||
77 | 59 | self.wall_time_func = wall_time_func | ||
78 | 60 | |||
79 | 61 | self.m1_rate = Ewma.one_minute_ewma() | ||
80 | 62 | self.m5_rate = Ewma.five_minute_ewma() | ||
81 | 63 | self.m15_rate = Ewma.fifteen_minute_ewma() | ||
82 | 64 | self.count = 0 | ||
83 | 65 | self.start_time = self.wall_time_func() | ||
84 | 66 | |||
85 | 67 | def mark(self, value=1): | ||
86 | 68 | """Mark the occurrence of a given number of events.""" | ||
87 | 69 | self.count += value | ||
88 | 70 | self.m1_rate.update(value) | ||
89 | 71 | self.m5_rate.update(value) | ||
90 | 72 | self.m15_rate.update(value) | ||
91 | 73 | |||
92 | 74 | def tick(self): | ||
93 | 75 | """Updates the moving averages.""" | ||
94 | 76 | self.m1_rate.tick() | ||
95 | 77 | self.m5_rate.tick() | ||
96 | 78 | self.m15_rate.tick() | ||
97 | 79 | |||
98 | 80 | def report(self, timestamp): | ||
99 | 81 | return MeterMetricReporter.MESSAGE % { | ||
100 | 82 | "key": self.name, | ||
101 | 83 | "count": self.count, | ||
102 | 84 | "mean_rate": self.mean_rate(), | ||
103 | 85 | "rate_1min": self.one_minute_rate(), | ||
104 | 86 | "rate_5min": self.five_minute_rate(), | ||
105 | 87 | "rate_15min": self.fifteen_minute_rate(), | ||
106 | 88 | "timestamp": timestamp} | ||
107 | 89 | |||
108 | 90 | def fifteen_minute_rate(self): | ||
109 | 91 | return self.m15_rate.rate | ||
110 | 92 | |||
111 | 93 | def five_minute_rate(self): | ||
112 | 94 | return self.m5_rate.rate | ||
113 | 95 | |||
114 | 96 | def one_minute_rate(self): | ||
115 | 97 | return self.m1_rate.rate | ||
116 | 98 | |||
117 | 99 | def mean_rate(self): | ||
118 | 100 | if self.count == 0: | ||
119 | 101 | return 0.0 | ||
120 | 102 | else: | ||
121 | 103 | elapsed = self.wall_time_func() - self.start_time | ||
122 | 104 | return float(self.count) / elapsed | ||
123 | 0 | 105 | ||
124 | === modified file 'txstatsd/metrics/metrics.py' | |||
125 | --- txstatsd/metrics/metrics.py 2011-07-27 15:02:32 +0000 | |||
126 | +++ txstatsd/metrics/metrics.py 2011-08-26 12:28:49 +0000 | |||
127 | @@ -1,5 +1,6 @@ | |||
128 | 1 | 1 | ||
129 | 2 | from txstatsd.metrics.gaugemetric import GaugeMetric | 2 | from txstatsd.metrics.gaugemetric import GaugeMetric |
130 | 3 | from txstatsd.metrics.metermetric import MeterMetric | ||
131 | 3 | from txstatsd.metrics.metric import Metric | 4 | from txstatsd.metrics.metric import Metric |
132 | 4 | 5 | ||
133 | 5 | 6 | ||
134 | @@ -28,6 +29,16 @@ | |||
135 | 28 | self._metrics[name] = gauge_metric | 29 | self._metrics[name] = gauge_metric |
136 | 29 | self._metrics[name].mark(value) | 30 | self._metrics[name].mark(value) |
137 | 30 | 31 | ||
138 | 32 | def meter(self, name, value, sample_rate=1): | ||
139 | 33 | """Mark the occurrence of a given number of events.""" | ||
140 | 34 | name = self.fully_qualify_name(name) | ||
141 | 35 | if not name in self._metrics: | ||
142 | 36 | meter_metric = MeterMetric(self.connection, | ||
143 | 37 | name, | ||
144 | 38 | sample_rate) | ||
145 | 39 | self._metrics[name] = meter_metric | ||
146 | 40 | self._metrics[name].mark(value) | ||
147 | 41 | |||
148 | 31 | def increment(self, name, value=1, sample_rate=1): | 42 | def increment(self, name, value=1, sample_rate=1): |
149 | 32 | """Report and increase in name by count.""" | 43 | """Report and increase in name by count.""" |
150 | 33 | name = self.fully_qualify_name(name) | 44 | name = self.fully_qualify_name(name) |
151 | 34 | 45 | ||
152 | === modified file 'txstatsd/server/processor.py' | |||
153 | --- txstatsd/server/processor.py 2011-07-25 16:09:37 +0000 | |||
154 | +++ txstatsd/server/processor.py 2011-08-26 12:28:49 +0000 | |||
155 | @@ -5,6 +5,8 @@ | |||
156 | 5 | 5 | ||
157 | 6 | from twisted.python import log | 6 | from twisted.python import log |
158 | 7 | 7 | ||
159 | 8 | from txstatsd.metrics.metermetric import MeterMetricReporter | ||
160 | 9 | |||
161 | 8 | 10 | ||
162 | 9 | SPACES = re.compile("\s+") | 11 | SPACES = re.compile("\s+") |
163 | 10 | SLASHES = re.compile("\/+") | 12 | SLASHES = re.compile("\/+") |
164 | @@ -43,6 +45,7 @@ | |||
165 | 43 | self.timer_metrics = {} | 45 | self.timer_metrics = {} |
166 | 44 | self.counter_metrics = {} | 46 | self.counter_metrics = {} |
167 | 45 | self.gauge_metrics = deque() | 47 | self.gauge_metrics = deque() |
168 | 48 | self.meter_metrics = {} | ||
169 | 46 | 49 | ||
170 | 47 | def fail(self, message): | 50 | def fail(self, message): |
171 | 48 | """Log and discard malformed message.""" | 51 | """Log and discard malformed message.""" |
172 | @@ -72,6 +75,8 @@ | |||
173 | 72 | self.process_timer_metric(key, fields[0], message) | 75 | self.process_timer_metric(key, fields[0], message) |
174 | 73 | elif fields[1] == "g": | 76 | elif fields[1] == "g": |
175 | 74 | self.process_gauge_metric(key, fields[0], message) | 77 | self.process_gauge_metric(key, fields[0], message) |
176 | 78 | elif fields[1] == "m": | ||
177 | 79 | self.process_meter_metric(key, fields[0], message) | ||
178 | 75 | else: | 80 | else: |
179 | 76 | return self.fail(message) | 81 | return self.fail(message) |
180 | 77 | 82 | ||
181 | @@ -110,6 +115,21 @@ | |||
182 | 110 | except (TypeError, ValueError): | 115 | except (TypeError, ValueError): |
183 | 111 | self.fail(message) | 116 | self.fail(message) |
184 | 112 | 117 | ||
185 | 118 | def process_meter_metric(self, key, composite, message): | ||
186 | 119 | values = composite.split(":") | ||
187 | 120 | if not len(values) == 1: | ||
188 | 121 | return self.fail(message) | ||
189 | 122 | |||
190 | 123 | try: | ||
191 | 124 | value = float(values[0]) | ||
192 | 125 | except (TypeError, ValueError): | ||
193 | 126 | self.fail(message) | ||
194 | 127 | |||
195 | 128 | if not key in self.meter_metrics: | ||
196 | 129 | metric = MeterMetricReporter(key, self.time_function) | ||
197 | 130 | self.meter_metrics[key] = metric | ||
198 | 131 | self.meter_metrics[key].mark(value) | ||
199 | 132 | |||
200 | 113 | def flush(self, interval=10000, percent=90): | 133 | def flush(self, interval=10000, percent=90): |
201 | 114 | """ | 134 | """ |
202 | 115 | Flush all queued stats, computing a normalized count based on | 135 | Flush all queued stats, computing a normalized count based on |
203 | @@ -136,6 +156,11 @@ | |||
204 | 136 | messages.extend(gauge_metrics) | 156 | messages.extend(gauge_metrics) |
205 | 137 | num_stats += events | 157 | num_stats += events |
206 | 138 | 158 | ||
207 | 159 | meter_metrics, events = self.flush_meter_metrics(timestamp) | ||
208 | 160 | if events > 0: | ||
209 | 161 | messages.extend(meter_metrics) | ||
210 | 162 | num_stats += events | ||
211 | 163 | |||
212 | 139 | messages.append("statsd.numStats %s %s" % (num_stats, timestamp)) | 164 | messages.append("statsd.numStats %s %s" % (num_stats, timestamp)) |
213 | 140 | return messages | 165 | return messages |
214 | 141 | 166 | ||
215 | @@ -211,3 +236,17 @@ | |||
216 | 211 | self.gauge_metrics.clear() | 236 | self.gauge_metrics.clear() |
217 | 212 | 237 | ||
218 | 213 | return (metrics, events) | 238 | return (metrics, events) |
219 | 239 | |||
220 | 240 | def flush_meter_metrics(self, timestamp): | ||
221 | 241 | metrics = [] | ||
222 | 242 | events = 0 | ||
223 | 243 | for metric in self.meter_metrics.itervalues(): | ||
224 | 244 | message = metric.report(timestamp) | ||
225 | 245 | metrics.append(message) | ||
226 | 246 | events += 1 | ||
227 | 247 | |||
228 | 248 | return (metrics, events) | ||
229 | 249 | |||
230 | 250 | def update_metrics(self): | ||
231 | 251 | for metric in self.meter_metrics.itervalues(): | ||
232 | 252 | metric.tick() | ||
233 | 214 | 253 | ||
234 | === modified file 'txstatsd/service.py' | |||
235 | --- txstatsd/service.py 2011-07-27 20:31:33 +0000 | |||
236 | +++ txstatsd/service.py 2011-08-26 12:28:49 +0000 | |||
237 | @@ -117,6 +117,12 @@ | |||
238 | 117 | report_name.upper(), ()): | 117 | report_name.upper(), ()): |
239 | 118 | reporting.schedule(reporter, 10, metrics.gauge) | 118 | reporting.schedule(reporter, 10, metrics.gauge) |
240 | 119 | 119 | ||
241 | 120 | # Schedule updates for those metrics expecting to be | ||
242 | 121 | # periodically updated, for example the meter metric. | ||
243 | 122 | metrics_updater = ReportingService() | ||
244 | 123 | metrics_updater.setServiceParent(service) | ||
245 | 124 | metrics_updater.schedule(processor.update_metrics, 5, None) | ||
246 | 125 | |||
247 | 120 | factory = GraphiteClientFactory(processor, options["flush-interval"]) | 126 | factory = GraphiteClientFactory(processor, options["flush-interval"]) |
248 | 121 | client = TCPClient(options["carbon-cache-host"], | 127 | client = TCPClient(options["carbon-cache-host"], |
249 | 122 | options["carbon-cache-port"], | 128 | options["carbon-cache-port"], |
250 | 123 | 129 | ||
251 | === added directory 'txstatsd/stats' | |||
252 | === added file 'txstatsd/stats/__init__.py' | |||
253 | === added file 'txstatsd/stats/ewma.py' | |||
254 | --- txstatsd/stats/ewma.py 1970-01-01 00:00:00 +0000 | |||
255 | +++ txstatsd/stats/ewma.py 2011-08-26 12:28:49 +0000 | |||
256 | @@ -0,0 +1,70 @@ | |||
257 | 1 | |||
258 | 2 | """ | ||
259 | 3 | An exponentially-weighted moving average. | ||
260 | 4 | |||
261 | 5 | See: | ||
262 | 6 | - U{UNIX Load Average Part 1: How It Works | ||
263 | 7 | <http://www.teamquest.com/pdfs/whitepaper/ldavg1.pdf>} | ||
264 | 8 | - U{UNIX Load Average Part 2: Not Your Average Average | ||
265 | 9 | <http://www.teamquest.com/pdfs/whitepaper/ldavg2.pdf>} | ||
266 | 10 | """ | ||
267 | 11 | |||
268 | 12 | import math | ||
269 | 13 | |||
270 | 14 | |||
271 | 15 | class Ewma(object): | ||
272 | 16 | M1_ALPHA = 1 - math.exp(-5 / 60.0) | ||
273 | 17 | M5_ALPHA = 1 - math.exp(-5 / 60.0 / 5) | ||
274 | 18 | M15_ALPHA = 1 - math.exp(-5 / 60.0 / 15) | ||
275 | 19 | |||
276 | 20 | @classmethod | ||
277 | 21 | def one_minute_ewma(cls): | ||
278 | 22 | """ | ||
279 | 23 | Creates a new C{Ewma} which is equivalent to the UNIX one minute | ||
280 | 24 | load average and which expects to be ticked every 5 seconds. | ||
281 | 25 | """ | ||
282 | 26 | return Ewma(Ewma.M1_ALPHA, 5) | ||
283 | 27 | |||
284 | 28 | @classmethod | ||
285 | 29 | def five_minute_ewma(cls): | ||
286 | 30 | """ | ||
287 | 31 | Creates a new C{Ewma} which is equivalent to the UNIX five minute | ||
288 | 32 | load average and which expects to be ticked every 5 seconds. | ||
289 | 33 | """ | ||
290 | 34 | return Ewma(Ewma.M5_ALPHA, 5) | ||
291 | 35 | |||
292 | 36 | @classmethod | ||
293 | 37 | def fifteen_minute_ewma(cls): | ||
294 | 38 | """ | ||
295 | 39 | Creates a new C{Ewma} which is equivalent to the UNIX fifteen | ||
296 | 40 | minute load average and which expects to be ticked every 5 seconds. | ||
297 | 41 | """ | ||
298 | 42 | return Ewma(Ewma.M15_ALPHA, 5) | ||
299 | 43 | |||
300 | 44 | def __init__(self, alpha, interval): | ||
301 | 45 | """Create a new C{Ewma} with a specific smoothing constant. | ||
302 | 46 | |||
303 | 47 | @param alpha: The smoothing constant. | ||
304 | 48 | @param interval: The expected tick interval in seconds. | ||
305 | 49 | """ | ||
306 | 50 | self.interval = interval | ||
307 | 51 | self.alpha = float(alpha) | ||
308 | 52 | |||
309 | 53 | self.initialized = False | ||
310 | 54 | self.rate = 0.0 | ||
311 | 55 | self.uncounted = 0 | ||
312 | 56 | |||
313 | 57 | def update(self, n): | ||
314 | 58 | """Update the moving average with a new value.""" | ||
315 | 59 | self.uncounted += n | ||
316 | 60 | |||
317 | 61 | def tick(self): | ||
318 | 62 | """Mark the passage of time and decay the current rate accordingly.""" | ||
319 | 63 | count = self.uncounted | ||
320 | 64 | self.uncounted = 0 | ||
321 | 65 | instant_rate = float(count) / self.interval | ||
322 | 66 | if self.initialized: | ||
323 | 67 | self.rate += (self.alpha * (instant_rate - self.rate)) | ||
324 | 68 | else: | ||
325 | 69 | self.rate = instant_rate | ||
326 | 70 | self.initialized = True | ||
327 | 0 | 71 | ||
328 | === added directory 'txstatsd/tests/stats' | |||
329 | === added file 'txstatsd/tests/stats/__init__.py' | |||
330 | === added file 'txstatsd/tests/stats/test_ewma.py' | |||
331 | --- txstatsd/tests/stats/test_ewma.py 1970-01-01 00:00:00 +0000 | |||
332 | +++ txstatsd/tests/stats/test_ewma.py 2011-08-26 12:28:49 +0000 | |||
333 | @@ -0,0 +1,315 @@ | |||
334 | 1 | |||
335 | 2 | import math | ||
336 | 3 | from unittest import TestCase | ||
337 | 4 | |||
338 | 5 | from txstatsd.stats.ewma import Ewma | ||
339 | 6 | |||
340 | 7 | |||
341 | 8 | def mark_minutes(minutes, ewma): | ||
342 | 9 | for i in range(1, minutes * 60, 5): | ||
343 | 10 | ewma.tick() | ||
344 | 11 | |||
345 | 12 | class TestEwmaOneMinute(TestCase): | ||
346 | 13 | def setUp(self): | ||
347 | 14 | self.ewma = Ewma.one_minute_ewma() | ||
348 | 15 | self.ewma.update(3) | ||
349 | 16 | self.ewma.tick() | ||
350 | 17 | |||
351 | 18 | def test_first_tick(self): | ||
352 | 19 | self.assertTrue( | ||
353 | 20 | (math.fabs(self.ewma.rate - 0.6) < 0.000001), | ||
354 | 21 | 'Should have a rate of 0.6 events/sec after the first tick') | ||
355 | 22 | |||
356 | 23 | def test_one_minute(self): | ||
357 | 24 | mark_minutes(1, self.ewma) | ||
358 | 25 | self.assertTrue( | ||
359 | 26 | (math.fabs(self.ewma.rate - 0.22072766) < 0.00000001), | ||
360 | 27 | 'Should have a rate of 0.22072766 events/sec after 1 minute') | ||
361 | 28 | |||
362 | 29 | def test_two_minutes(self): | ||
363 | 30 | mark_minutes(2, self.ewma) | ||
364 | 31 | self.assertTrue( | ||
365 | 32 | (math.fabs(self.ewma.rate - 0.08120117) < 0.00000001), | ||
366 | 33 | 'Should have a rate of 0.08120117 events/sec after 2 minutes') | ||
367 | 34 | |||
368 | 35 | def test_three_minutes(self): | ||
369 | 36 | mark_minutes(3, self.ewma) | ||
370 | 37 | self.assertTrue( | ||
371 | 38 | (math.fabs(self.ewma.rate - 0.02987224) < 0.00000001), | ||
372 | 39 | 'Should have a rate of 0.02987224 events/sec after 3 minutes') | ||
373 | 40 | |||
374 | 41 | def test_four_minutes(self): | ||
375 | 42 | mark_minutes(4, self.ewma) | ||
376 | 43 | self.assertTrue( | ||
377 | 44 | (math.fabs(self.ewma.rate - 0.01098938) < 0.00000001), | ||
378 | 45 | 'Should have a rate of 0.01098938 events/sec after 4 minutes') | ||
379 | 46 | |||
380 | 47 | def test_five_minutes(self): | ||
381 | 48 | mark_minutes(5, self.ewma) | ||
382 | 49 | self.assertTrue( | ||
383 | 50 | (math.fabs(self.ewma.rate - 0.00404277) < 0.00000001), | ||
384 | 51 | 'Should have a rate of 0.00404277 events/sec after 5 minutes') | ||
385 | 52 | |||
386 | 53 | def test_six_minutes(self): | ||
387 | 54 | mark_minutes(6, self.ewma) | ||
388 | 55 | self.assertTrue( | ||
389 | 56 | (math.fabs(self.ewma.rate - 0.00148725) < 0.00000001), | ||
390 | 57 | 'Should have a rate of 0.00148725 events/sec after 6 minutes') | ||
391 | 58 | |||
392 | 59 | def test_seven_minutes(self): | ||
393 | 60 | mark_minutes(7, self.ewma) | ||
394 | 61 | self.assertTrue( | ||
395 | 62 | (math.fabs(self.ewma.rate - 0.00054713) < 0.00000001), | ||
396 | 63 | 'Should have a rate of 0.00054713 events/sec after 7 minutes') | ||
397 | 64 | |||
398 | 65 | def test_eight_minutes(self): | ||
399 | 66 | mark_minutes(8, self.ewma) | ||
400 | 67 | self.assertTrue( | ||
401 | 68 | (math.fabs(self.ewma.rate - 0.00020128) < 0.00000001), | ||
402 | 69 | 'Should have a rate of 0.00020128 events/sec after 8 minutes') | ||
403 | 70 | |||
404 | 71 | def test_nine_minutes(self): | ||
405 | 72 | mark_minutes(9, self.ewma) | ||
406 | 73 | self.assertTrue( | ||
407 | 74 | (math.fabs(self.ewma.rate - 0.00007405) < 0.00000001), | ||
408 | 75 | 'Should have a rate of 0.00007405 events/sec after 9 minutes') | ||
409 | 76 | |||
410 | 77 | def test_ten_minutes(self): | ||
411 | 78 | mark_minutes(10, self.ewma) | ||
412 | 79 | self.assertTrue( | ||
413 | 80 | (math.fabs(self.ewma.rate - 0.00002724) < 0.00000001), | ||
414 | 81 | 'Should have a rate of 0.00002724 events/sec after 10 minutes') | ||
415 | 82 | |||
416 | 83 | def test_eleven_minutes(self): | ||
417 | 84 | mark_minutes(11, self.ewma) | ||
418 | 85 | self.assertTrue( | ||
419 | 86 | (math.fabs(self.ewma.rate - 0.00001002) < 0.00000001), | ||
420 | 87 | 'Should have a rate of 0.00001002 events/sec after 11 minutes') | ||
421 | 88 | |||
422 | 89 | def test_twelve_minutes(self): | ||
423 | 90 | mark_minutes(12, self.ewma) | ||
424 | 91 | self.assertTrue( | ||
425 | 92 | (math.fabs(self.ewma.rate - 0.00000369) < 0.00000001), | ||
426 | 93 | 'Should have a rate of 0.00000369 events/sec after 12 minutes') | ||
427 | 94 | |||
428 | 95 | def test_thirteen_minutes(self): | ||
429 | 96 | mark_minutes(13, self.ewma) | ||
430 | 97 | self.assertTrue( | ||
431 | 98 | (math.fabs(self.ewma.rate - 0.00000136) < 0.00000001), | ||
432 | 99 | 'Should have a rate of 0.00000136 events/sec after 13 minutes') | ||
433 | 100 | |||
434 | 101 | def test_fourteen_minutes(self): | ||
435 | 102 | mark_minutes(14, self.ewma) | ||
436 | 103 | self.assertTrue( | ||
437 | 104 | (math.fabs(self.ewma.rate - 0.00000050) < 0.00000001), | ||
438 | 105 | 'Should have a rate of 0.00000050 events/sec after 14 minutes') | ||
439 | 106 | |||
440 | 107 | def test_fifteen_minutes(self): | ||
441 | 108 | mark_minutes(15, self.ewma) | ||
442 | 109 | self.assertTrue( | ||
443 | 110 | (math.fabs(self.ewma.rate - 0.00000018) < 0.00000001), | ||
444 | 111 | 'Should have a rate of 0.00000018 events/sec after 15 minutes') | ||
445 | 112 | |||
446 | 113 | |||
447 | 114 | class TestEwmaFiveMinute(TestCase): | ||
448 | 115 | def setUp(self): | ||
449 | 116 | self.ewma = Ewma.five_minute_ewma() | ||
450 | 117 | self.ewma.update(3) | ||
451 | 118 | self.ewma.tick() | ||
452 | 119 | |||
453 | 120 | def test_first_tick(self): | ||
454 | 121 | self.assertTrue( | ||
455 | 122 | (math.fabs(self.ewma.rate - 0.6) < 0.000001), | ||
456 | 123 | 'Should have a rate of 0.6 events/sec after the first tick') | ||
457 | 124 | |||
458 | 125 | def test_one_minute(self): | ||
459 | 126 | mark_minutes(1, self.ewma) | ||
460 | 127 | self.assertTrue( | ||
461 | 128 | (math.fabs(self.ewma.rate - 0.49123845) < 0.00000001), | ||
462 | 129 | 'Should have a rate of 0.49123845 events/sec after 1 minute') | ||
463 | 130 | |||
464 | 131 | def test_two_minutes(self): | ||
465 | 132 | mark_minutes(2, self.ewma) | ||
466 | 133 | self.assertTrue( | ||
467 | 134 | (math.fabs(self.ewma.rate - 0.40219203) < 0.00000001), | ||
468 | 135 | 'Should have a rate of 0.40219203 events/sec after 2 minutes') | ||
469 | 136 | |||
470 | 137 | def test_three_minutes(self): | ||
471 | 138 | mark_minutes(3, self.ewma) | ||
472 | 139 | self.assertTrue( | ||
473 | 140 | (math.fabs(self.ewma.rate - 0.32928698) < 0.00000001), | ||
474 | 141 | 'Should have a rate of 0.32928698 events/sec after 3 minutes') | ||
475 | 142 | |||
476 | 143 | def test_four_minutes(self): | ||
477 | 144 | mark_minutes(4, self.ewma) | ||
478 | 145 | self.assertTrue( | ||
479 | 146 | (math.fabs(self.ewma.rate - 0.26959738) < 0.00000001), | ||
480 | 147 | 'Should have a rate of 0.26959738 events/sec after 4 minutes') | ||
481 | 148 | |||
482 | 149 | def test_five_minutes(self): | ||
483 | 150 | mark_minutes(5, self.ewma) | ||
484 | 151 | self.assertTrue( | ||
485 | 152 | (math.fabs(self.ewma.rate - 0.22072766) < 0.00000001), | ||
486 | 153 | 'Should have a rate of 0.22072766 events/sec after 5 minutes') | ||
487 | 154 | |||
488 | 155 | def test_six_minutes(self): | ||
489 | 156 | mark_minutes(6, self.ewma) | ||
490 | 157 | self.assertTrue( | ||
491 | 158 | (math.fabs(self.ewma.rate - 0.18071653) < 0.00000001), | ||
492 | 159 | 'Should have a rate of 0.18071653 events/sec after 6 minutes') | ||
493 | 160 | |||
494 | 161 | def test_seven_minutes(self): | ||
495 | 162 | mark_minutes(7, self.ewma) | ||
496 | 163 | self.assertTrue( | ||
497 | 164 | (math.fabs(self.ewma.rate - 0.14795818) < 0.00000001), | ||
498 | 165 | 'Should have a rate of 0.14795818 events/sec after 7 minutes') | ||
499 | 166 | |||
500 | 167 | def test_eight_minutes(self): | ||
501 | 168 | mark_minutes(8, self.ewma) | ||
502 | 169 | self.assertTrue( | ||
503 | 170 | (math.fabs(self.ewma.rate - 0.12113791) < 0.00000001), | ||
504 | 171 | 'Should have a rate of 0.12113791 events/sec after 8 minutes') | ||
505 | 172 | |||
506 | 173 | def test_nine_minutes(self): | ||
507 | 174 | mark_minutes(9, self.ewma) | ||
508 | 175 | self.assertTrue( | ||
509 | 176 | (math.fabs(self.ewma.rate - 0.09917933) < 0.00000001), | ||
510 | 177 | 'Should have a rate of 0.09917933 events/sec after 9 minutes') | ||
511 | 178 | |||
512 | 179 | def test_ten_minutes(self): | ||
513 | 180 | mark_minutes(10, self.ewma) | ||
514 | 181 | self.assertTrue( | ||
515 | 182 | (math.fabs(self.ewma.rate - 0.08120117) < 0.00000001), | ||
516 | 183 | 'Should have a rate of 0.08120117 events/sec after 10 minutes') | ||
517 | 184 | |||
518 | 185 | def test_eleven_minutes(self): | ||
519 | 186 | mark_minutes(11, self.ewma) | ||
520 | 187 | self.assertTrue( | ||
521 | 188 | (math.fabs(self.ewma.rate - 0.06648190) < 0.00000001), | ||
522 | 189 | 'Should have a rate of 0.06648190 events/sec after 11 minutes') | ||
523 | 190 | |||
524 | 191 | def test_twelve_minutes(self): | ||
525 | 192 | mark_minutes(12, self.ewma) | ||
526 | 193 | self.assertTrue( | ||
527 | 194 | (math.fabs(self.ewma.rate - 0.05443077) < 0.00000001), | ||
528 | 195 | 'Should have a rate of 0.05443077 events/sec after 12 minutes') | ||
529 | 196 | |||
530 | 197 | def test_thirteen_minutes(self): | ||
531 | 198 | mark_minutes(13, self.ewma) | ||
532 | 199 | self.assertTrue( | ||
533 | 200 | (math.fabs(self.ewma.rate - 0.04456415) < 0.00000001), | ||
534 | 201 | 'Should have a rate of 0.04456415 events/sec after 13 minutes') | ||
535 | 202 | |||
536 | 203 | def test_fourteen_minutes(self): | ||
537 | 204 | mark_minutes(14, self.ewma) | ||
538 | 205 | self.assertTrue( | ||
539 | 206 | (math.fabs(self.ewma.rate - 0.03648604) < 0.00000001), | ||
540 | 207 | 'Should have a rate of 0.03648604 events/sec after 14 minutes') | ||
541 | 208 | |||
542 | 209 | def test_fifteen_minutes(self): | ||
543 | 210 | mark_minutes(15, self.ewma) | ||
544 | 211 | self.assertTrue( | ||
545 | 212 | (math.fabs(self.ewma.rate - 0.02987224) < 0.00000001), | ||
546 | 213 | 'Should have a rate of 0.02987224 events/sec after 15 minutes') | ||
547 | 214 | |||
548 | 215 | |||
549 | 216 | class TestEwmaFifteenMinute(TestCase): | ||
550 | 217 | def setUp(self): | ||
551 | 218 | self.ewma = Ewma.fifteen_minute_ewma() | ||
552 | 219 | self.ewma.update(3) | ||
553 | 220 | self.ewma.tick() | ||
554 | 221 | |||
555 | 222 | def test_first_tick(self): | ||
556 | 223 | self.assertTrue( | ||
557 | 224 | (math.fabs(self.ewma.rate - 0.6) < 0.000001), | ||
558 | 225 | 'Should have a rate of 0.6 events/sec after the first tick') | ||
559 | 226 | |||
560 | 227 | def test_one_minute(self): | ||
561 | 228 | mark_minutes(1, self.ewma) | ||
562 | 229 | self.assertTrue( | ||
563 | 230 | (math.fabs(self.ewma.rate - 0.56130419) < 0.00000001), | ||
564 | 231 | 'Should have a rate of 0.56130419 events/sec after 1 minute') | ||
565 | 232 | |||
566 | 233 | def test_two_minutes(self): | ||
567 | 234 | mark_minutes(2, self.ewma) | ||
568 | 235 | self.assertTrue( | ||
569 | 236 | (math.fabs(self.ewma.rate - 0.52510399) < 0.00000001), | ||
570 | 237 | 'Should have a rate of 0.52510399 events/sec after 2 minutes') | ||
571 | 238 | |||
572 | 239 | def test_three_minutes(self): | ||
573 | 240 | mark_minutes(3, self.ewma) | ||
574 | 241 | self.assertTrue( | ||
575 | 242 | (math.fabs(self.ewma.rate - 0.49123845) < 0.00000001), | ||
576 | 243 | 'Should have a rate of 0.49123845 events/sec after 3 minutes') | ||
577 | 244 | |||
578 | 245 | def test_four_minutes(self): | ||
579 | 246 | mark_minutes(4, self.ewma) | ||
580 | 247 | self.assertTrue( | ||
581 | 248 | (math.fabs(self.ewma.rate - 0.45955700) < 0.00000001), | ||
582 | 249 | 'Should have a rate of 0.45955700 events/sec after 4 minutes') | ||
583 | 250 | |||
584 | 251 | def test_five_minutes(self): | ||
585 | 252 | mark_minutes(5, self.ewma) | ||
586 | 253 | self.assertTrue( | ||
587 | 254 | (math.fabs(self.ewma.rate - 0.42991879) < 0.00000001), | ||
588 | 255 | 'Should have a rate of 0.42991879 events/sec after 5 minutes') | ||
589 | 256 | |||
590 | 257 | def test_six_minutes(self): | ||
591 | 258 | mark_minutes(6, self.ewma) | ||
592 | 259 | self.assertTrue( | ||
593 | 260 | (math.fabs(self.ewma.rate - 0.40219203) < 0.00000001), | ||
594 | 261 | 'Should have a rate of 0.40219203 events/sec after 6 minutes') | ||
595 | 262 | |||
596 | 263 | def test_seven_minutes(self): | ||
597 | 264 | mark_minutes(7, self.ewma) | ||
598 | 265 | self.assertTrue( | ||
599 | 266 | (math.fabs(self.ewma.rate - 0.37625345) < 0.00000001), | ||
600 | 267 | 'Should have a rate of 0.37625345 events/sec after 7 minutes') | ||
601 | 268 | |||
602 | 269 | def test_eight_minutes(self): | ||
603 | 270 | mark_minutes(8, self.ewma) | ||
604 | 271 | self.assertTrue( | ||
605 | 272 | (math.fabs(self.ewma.rate - 0.35198773) < 0.00000001), | ||
606 | 273 | 'Should have a rate of 0.35198773 events/sec after 8 minutes') | ||
607 | 274 | |||
608 | 275 | def test_nine_minutes(self): | ||
609 | 276 | mark_minutes(9, self.ewma) | ||
610 | 277 | self.assertTrue( | ||
611 | 278 | (math.fabs(self.ewma.rate - 0.32928698) < 0.00000001), | ||
612 | 279 | 'Should have a rate of 0.32928698 events/sec after 9 minutes') | ||
613 | 280 | |||
614 | 281 | def test_ten_minutes(self): | ||
615 | 282 | mark_minutes(10, self.ewma) | ||
616 | 283 | self.assertTrue( | ||
617 | 284 | (math.fabs(self.ewma.rate - 0.30805027) < 0.00000001), | ||
618 | 285 | 'Should have a rate of 0.30805027 events/sec after 10 minutes') | ||
619 | 286 | |||
620 | 287 | def test_eleven_minutes(self): | ||
621 | 288 | mark_minutes(11, self.ewma) | ||
622 | 289 | self.assertTrue( | ||
623 | 290 | (math.fabs(self.ewma.rate - 0.28818318) < 0.00000001), | ||
624 | 291 | 'Should have a rate of 0.28818318 events/sec after 11 minutes') | ||
625 | 292 | |||
626 | 293 | def test_twelve_minutes(self): | ||
627 | 294 | mark_minutes(12, self.ewma) | ||
628 | 295 | self.assertTrue( | ||
629 | 296 | (math.fabs(self.ewma.rate - 0.26959738) < 0.00000001), | ||
630 | 297 | 'Should have a rate of 0.26959738 events/sec after 12 minutes') | ||
631 | 298 | |||
632 | 299 | def test_thirteen_minutes(self): | ||
633 | 300 | mark_minutes(13, self.ewma) | ||
634 | 301 | self.assertTrue( | ||
635 | 302 | (math.fabs(self.ewma.rate - 0.25221023) < 0.00000001), | ||
636 | 303 | 'Should have a rate of 0.25221023 events/sec after 13 minutes') | ||
637 | 304 | |||
638 | 305 | def test_fourteen_minutes(self): | ||
639 | 306 | mark_minutes(14, self.ewma) | ||
640 | 307 | self.assertTrue( | ||
641 | 308 | (math.fabs(self.ewma.rate - 0.23594443) < 0.00000001), | ||
642 | 309 | 'Should have a rate of 0.23594443 events/sec after 14 minutes') | ||
643 | 310 | |||
644 | 311 | def test_fifteen_minutes(self): | ||
645 | 312 | mark_minutes(15, self.ewma) | ||
646 | 313 | self.assertTrue( | ||
647 | 314 | (math.fabs(self.ewma.rate - 0.22072766) < 0.00000001), | ||
648 | 315 | 'Should have a rate of 0.22072766 events/sec after 15 minutes') | ||
649 | 0 | 316 | ||
650 | === modified file 'txstatsd/tests/test_metrics.py' | |||
651 | --- txstatsd/tests/test_metrics.py 2011-07-03 11:49:05 +0000 | |||
652 | +++ txstatsd/tests/test_metrics.py 2011-08-26 12:28:49 +0000 | |||
653 | @@ -32,6 +32,12 @@ | |||
654 | 32 | self.assertEqual(self.connection.data, | 32 | self.assertEqual(self.connection.data, |
655 | 33 | 'txstatsd.tests.gauge:102|g') | 33 | 'txstatsd.tests.gauge:102|g') |
656 | 34 | 34 | ||
657 | 35 | def test_meter(self): | ||
658 | 36 | """Test reporting of a meter metric sample.""" | ||
659 | 37 | self.metrics.meter('meter', 3) | ||
660 | 38 | self.assertEqual(self.connection.data, | ||
661 | 39 | 'txstatsd.tests.meter:3|m') | ||
662 | 40 | |||
663 | 35 | def test_counter(self): | 41 | def test_counter(self): |
664 | 36 | """Test the increment and decrement operations.""" | 42 | """Test the increment and decrement operations.""" |
665 | 37 | self.metrics.increment('counter', 18) | 43 | self.metrics.increment('counter', 18) |
666 | 38 | 44 | ||
667 | === modified file 'txstatsd/tests/test_processor.py' | |||
668 | --- txstatsd/tests/test_processor.py 2011-07-04 03:00:04 +0000 | |||
669 | +++ txstatsd/tests/test_processor.py 2011-08-26 12:28:49 +0000 | |||
670 | @@ -1,3 +1,5 @@ | |||
671 | 1 | import time | ||
672 | 2 | |||
673 | 1 | from unittest import TestCase | 3 | from unittest import TestCase |
674 | 2 | 4 | ||
675 | 3 | from txstatsd.server.processor import MessageProcessor | 5 | from txstatsd.server.processor import MessageProcessor |
676 | @@ -224,3 +226,73 @@ | |||
677 | 224 | self.assertEqual( | 226 | self.assertEqual( |
678 | 225 | "statsd.numStats 1 42", messages[1]) | 227 | "statsd.numStats 1 42", messages[1]) |
679 | 226 | self.assertEqual(0, len(self.processor.gauge_metrics)) | 228 | self.assertEqual(0, len(self.processor.gauge_metrics)) |
680 | 229 | |||
681 | 230 | |||
682 | 231 | class FlushMeterMetricMessagesTest(TestCase): | ||
683 | 232 | |||
684 | 233 | def setUp(self): | ||
685 | 234 | self.processor = MessageProcessor(time_function=self.wall_clock_time) | ||
686 | 235 | self.time_now = int(time.time()) | ||
687 | 236 | |||
688 | 237 | def wall_clock_time(self): | ||
689 | 238 | return self.time_now | ||
690 | 239 | |||
691 | 240 | def mark_minutes(self, minutes): | ||
692 | 241 | for i in range(1, minutes * 60, 5): | ||
693 | 242 | self.processor.update_metrics() | ||
694 | 243 | |||
695 | 244 | def test_flush_meter_metric(self): | ||
696 | 245 | """ | ||
697 | 246 | Test the correct rendering of the Graphite report for | ||
698 | 247 | a meter metric. | ||
699 | 248 | """ | ||
700 | 249 | self.processor.process("gorets:3.0|m") | ||
701 | 250 | |||
702 | 251 | self.time_now += 1 | ||
703 | 252 | messages = self.processor.flush() | ||
704 | 253 | self.assertEqual(2, len(messages)) | ||
705 | 254 | meter_metric = messages[0].splitlines() | ||
706 | 255 | self.assertEqual( | ||
707 | 256 | "stats.meter.gorets.count 3.0 %s" % self.time_now, | ||
708 | 257 | meter_metric[0]) | ||
709 | 258 | self.assertEqual( | ||
710 | 259 | "stats.meter.gorets.mean_rate 3.0 %s" % self.time_now, | ||
711 | 260 | meter_metric[1]) | ||
712 | 261 | self.assertEqual( | ||
713 | 262 | "stats.meter.gorets.1min_rate 0.0 %s" % self.time_now, | ||
714 | 263 | meter_metric[2]) | ||
715 | 264 | self.assertEqual( | ||
716 | 265 | "stats.meter.gorets.5min_rate 0.0 %s" % self.time_now, | ||
717 | 266 | meter_metric[3]) | ||
718 | 267 | self.assertEqual( | ||
719 | 268 | "stats.meter.gorets.15min_rate 0.0 %s" % self.time_now, | ||
720 | 269 | meter_metric[4]) | ||
721 | 270 | self.assertEqual( | ||
722 | 271 | "statsd.numStats 1 %s" % self.time_now, messages[1]) | ||
723 | 272 | |||
724 | 273 | # As we are employing the expected results from test_ewma.py | ||
725 | 274 | # we perform the initial tick(), before advancing the clock 60sec. | ||
726 | 275 | self.processor.update_metrics() | ||
727 | 276 | |||
728 | 277 | self.mark_minutes(1) | ||
729 | 278 | self.time_now += 60 | ||
730 | 279 | messages = self.processor.flush() | ||
731 | 280 | self.assertEqual(2, len(messages)) | ||
732 | 281 | meter_metric = messages[0].splitlines() | ||
733 | 282 | self.assertEqual( | ||
734 | 283 | "stats.meter.gorets.count 3.0 %s" % self.time_now, | ||
735 | 284 | meter_metric[0]) | ||
736 | 285 | self.assertTrue( | ||
737 | 286 | meter_metric[1].startswith( | ||
738 | 287 | "stats.meter.gorets.mean_rate 0.04918032")) | ||
739 | 288 | self.assertTrue( | ||
740 | 289 | meter_metric[2].startswith( | ||
741 | 290 | "stats.meter.gorets.1min_rate 0.22072766")) | ||
742 | 291 | self.assertTrue( | ||
743 | 292 | meter_metric[3].startswith( | ||
744 | 293 | "stats.meter.gorets.5min_rate 0.49123845")) | ||
745 | 294 | self.assertTrue( | ||
746 | 295 | meter_metric[4].startswith( | ||
747 | 296 | "stats.meter.gorets.15min_rate 0.5613041")) | ||
748 | 297 | self.assertEqual( | ||
749 | 298 | "statsd.numStats 1 %s" % self.time_now, messages[1]) | ||
750 | 227 | 299 | ||
751 | === modified file 'txstatsd/version.py' | |||
752 | --- txstatsd/version.py 2011-08-23 02:05:21 +0000 | |||
753 | +++ txstatsd/version.py 2011-08-26 12:28:49 +0000 | |||
754 | @@ -1,1 +1,1 @@ | |||
756 | 1 | txstatsd = "0.1.3" | 1 | txstatsd = "0.2.0" |
im under the impression that those would be better implemented as functions on the graphite side, but other than that looks good to me. +1