Merge ~twom/launchpad:stats-code-import-stats into launchpad:master

Proposed by Tom Wardill
Status: Merged
Approved by: Tom Wardill
Approved revision: bb6163ca342dfad7fc29cc7cd4ac4feb5a91d73e
Merge reported by: Otto Co-Pilot
Merged at revision: not available
Proposed branch: ~twom/launchpad:stats-code-import-stats
Merge into: launchpad:master
Diff against target: 160 lines (+91/-0)
2 files modified
lib/lp/services/statsd/numbercruncher.py (+27/-0)
lib/lp/services/statsd/tests/test_numbercruncher.py (+64/-0)
Reviewer Review Type Date Requested Status
Colin Watson Approve
Review via email: mp+397427@code.launchpad.net

Commit message

Add CodeImportJob stats to numbercruncher

To post a comment you must log in.
Revision history for this message
Colin Watson (cjwatson) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/lib/lp/services/statsd/numbercruncher.py b/lib/lp/services/statsd/numbercruncher.py
2index 3f1a6e1..b143927 100644
3--- a/lib/lp/services/statsd/numbercruncher.py
4+++ b/lib/lp/services/statsd/numbercruncher.py
5@@ -8,8 +8,10 @@ from __future__ import absolute_import, print_function, unicode_literals
6 __metaclass__ = type
7 __all__ = ['NumberCruncher']
8
9+from datetime import datetime
10 import logging
11
12+import pytz
13 from storm.expr import (
14 Count,
15 Sum,
16@@ -27,6 +29,8 @@ from zope.component import getUtility
17 from lp.buildmaster.enums import BuilderCleanStatus
18 from lp.buildmaster.interfaces.builder import IBuilderSet
19 from lp.buildmaster.manager import PrefetchedBuilderFactory
20+from lp.code.enums import CodeImportJobState
21+from lp.code.model.codeimportjob import CodeImportJob
22 from lp.services.database.interfaces import IStore
23 from lp.services.librarian.model import LibraryFileContent
24 from lp.services.statsd.interfaces.statsd_client import IStatsdClient
25@@ -40,6 +44,7 @@ class NumberCruncher(service.Service):
26 QUEUE_INTERVAL = 60
27 BUILDER_INTERVAL = 60
28 LIBRARIAN_INTERVAL = 3600
29+ CODE_IMPORT_INTERVAL = 60
30
31 def __init__(self, clock=None, builder_factory=None):
32 if clock is None:
33@@ -155,6 +160,27 @@ class NumberCruncher(service.Service):
34 self.logger.exception("Failure while updating librarian stats:")
35 transaction.abort()
36
37+ def updateCodeImportStats(self):
38+ """Update stats about code imports.
39+
40+ This aborts the current transaction before returning.
41+ """
42+ try:
43+ self.logger.debug("Update code import stats.")
44+ store = IStore(CodeImportJob)
45+ pending = store.find(
46+ CodeImportJob,
47+ CodeImportJob.state == CodeImportJobState.PENDING).count()
48+ overdue = store.find(
49+ CodeImportJob,
50+ CodeImportJob.date_due < datetime.now(pytz.UTC)).count()
51+ self._sendGauge("codeimport.pending", pending)
52+ self._sendGauge("codeimport.overdue", overdue)
53+ self.logger.debug("Code import stats update complete")
54+ except Exception:
55+ self.logger.exception("Failure while updating code import stats.")
56+ transaction.abort()
57+
58 def startService(self):
59 self.logger.info("Starting number-cruncher service.")
60 self.loops = []
61@@ -163,6 +189,7 @@ class NumberCruncher(service.Service):
62 (self.QUEUE_INTERVAL, self.updateBuilderQueues),
63 (self.BUILDER_INTERVAL, self.updateBuilderStats),
64 (self.LIBRARIAN_INTERVAL, self.updateLibrarianStats),
65+ (self.CODE_IMPORT_INTERVAL, self.updateCodeImportStats),
66 ):
67 loop, stopping_deferred = self._startLoop(interval, callback)
68 self.loops.append(loop)
69diff --git a/lib/lp/services/statsd/tests/test_numbercruncher.py b/lib/lp/services/statsd/tests/test_numbercruncher.py
70index 41b6bd4..f2ea44e 100644
71--- a/lib/lp/services/statsd/tests/test_numbercruncher.py
72+++ b/lib/lp/services/statsd/tests/test_numbercruncher.py
73@@ -17,12 +17,14 @@ from testtools.twistedsupport import AsynchronousDeferredRunTest
74 import transaction
75 from twisted.internet import task
76 from zope.component import getUtility
77+from zope.security.proxy import removeSecurityProxy
78
79 from lp.buildmaster.enums import BuilderCleanStatus
80 from lp.buildmaster.interactor import BuilderSlave
81 from lp.buildmaster.interfaces.processor import IProcessorSet
82 from lp.buildmaster.model.buildqueue import BuildQueue
83 from lp.buildmaster.tests.mock_slaves import OkSlave
84+from lp.code.enums import CodeImportJobState
85 from lp.services.database.isolation import is_transaction_in_progress
86 from lp.services.database.policy import DatabaseBlockedPolicy
87 from lp.services.log.logger import BufferLogger
88@@ -232,6 +234,57 @@ class TestNumberCruncher(StatsMixin, TestCaseWithFactory):
89 "Failure while updating librarian stats:",
90 cruncher.logger.getLogBuffer())
91
92+ def test_updateCodeImportStats(self):
93+ clock = task.Clock()
94+ cruncher = NumberCruncher(clock=clock)
95+ cruncher.updateCodeImportStats()
96+
97+ self.assertFalse(is_transaction_in_progress())
98+ self.assertEqual(2, self.stats_client.gauge.call_count)
99+ self.assertThat(
100+ [x[0] for x in self.stats_client.gauge.call_args_list],
101+ MatchesListwise([
102+ MatchesListwise([
103+ Equals('codeimport.pending,env=test'),
104+ Equals(1),
105+ ]),
106+ MatchesListwise([
107+ Equals('codeimport.overdue,env=test'),
108+ Equals(1),
109+ ]),
110+ ]))
111+
112+ job = removeSecurityProxy(self.factory.makeCodeImportJob())
113+ job.state = CodeImportJobState.PENDING
114+ self.stats_client.gauge.reset_mock()
115+ cruncher.updateCodeImportStats()
116+
117+ self.assertEqual(2, self.stats_client.gauge.call_count)
118+ self.assertThat(
119+ [x[0] for x in self.stats_client.gauge.call_args_list],
120+ MatchesListwise([
121+ MatchesListwise([
122+ Equals('codeimport.pending,env=test'),
123+ Equals(2),
124+ ]),
125+ MatchesListwise([
126+ Equals('codeimport.overdue,env=test'),
127+ Equals(2),
128+ ]),
129+ ]))
130+
131+ def test_updateCodeImportStats_error(self):
132+ clock = task.Clock()
133+ cruncher = NumberCruncher(clock=clock)
134+ cruncher.logger = BufferLogger()
135+ with DatabaseBlockedPolicy():
136+ cruncher.updateCodeImportStats()
137+
138+ self.assertFalse(is_transaction_in_progress())
139+ self.assertIn(
140+ "Failure while updating code import stats.",
141+ cruncher.logger.getLogBuffer())
142+
143 def test_startService_starts_update_queues_loop(self):
144 clock = task.Clock()
145 cruncher = NumberCruncher(clock=clock)
146@@ -264,3 +317,14 @@ class TestNumberCruncher(StatsMixin, TestCaseWithFactory):
147 advance = NumberCruncher.LIBRARIAN_INTERVAL + 1
148 clock.advance(advance)
149 self.assertNotEqual(0, cruncher.updateLibrarianStats.call_count)
150+
151+ def test_startService_starts_update_code_import_loop(self):
152+ clock = task.Clock()
153+ cruncher = NumberCruncher(clock=clock)
154+
155+ cruncher.updateCodeImportStats = FakeMethod()
156+
157+ cruncher.startService()
158+ advance = NumberCruncher.CODE_IMPORT_INTERVAL + 1
159+ clock.advance(advance)
160+ self.assertNotEqual(0, cruncher.updateCodeImportStats.call_count)

Subscribers

People subscribed via source and target branches

to status/vote changes: