Merge ~twom/launchpad:stats-daemon into launchpad:master

[Tom Wardill]

Split timed stats into separate daemon

Ideally we want generation/updating of stats to not block actual work. Move the timed stats generation out of buildd-manager to it's own daemon.
Leave the event driven stats in buildd-manager.

The implementation of the daemon here was heavily based on (and imports part of) buildd-manager itself, as that seemed the simplest method to allow reuse of the existing vitals generation code.

[Colin Watson (cjwatson)]

Overall this looks pretty good. It feels like the sort of thing that's likely to want to be restructured a bit once it covers more of the application, but that's fine.

review: Approve

6b2665a... by Tom Wardill on 2020-09-02

Fix typo

diff --git a/daemons/numbercruncher.tac b/daemons/numbercruncher.tac
new file mode 100644
index 0000000..e9a7bf7
--- /dev/null
+++ b/daemons/numbercruncher.tac
@@ -0,0 +1,35 @@
+# Copyright 2009-202 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+# Twisted Application Configuration file.
+# Use with "twistd2.4 -y <file.tac>", e.g. "twistd -noy server.tac"
+
+
+from twisted.application import service
+from twisted.scripts.twistd import ServerOptions
+
+from lp.services.daemons import readyservice
+from lp.services.scripts import execute_zcml_for_scripts
+from lp.services.statsd.numbercruncher import NumberCruncher
+from lp.services.twistedsupport.features import setup_feature_controller
+from lp.services.twistedsupport.loggingsupport import RotatableFileLogObserver
+
+execute_zcml_for_scripts()
+
+options = ServerOptions()
+options.parseOptions()
+
+application = service.Application('BuilddManager')
+application.addComponent(
+    RotatableFileLogObserver(options.get('logfile')), ignoreClass=1)
+
+# Service that announces when the daemon is ready.
+readyservice.ReadyService().setServiceParent(application)
+
+
+# Service for updating statsd receivers.
+service = NumberCruncher()
+service.setServiceParent(application)
+
+# Allow use of feature flags.
+setup_feature_controller('number-cruncher')
diff --git a/lib/lp/buildmaster/manager.py b/lib/lp/buildmaster/manager.py
index d0aa32e..7bbe4aa 100644
--- a/lib/lp/buildmaster/manager.py
+++ b/lib/lp/buildmaster/manager.py
@@ -8,6 +8,7 @@ __metaclass__ = type
 __all__ = [
     'BuilddManager',
     'BUILDD_MANAGER_LOG_NAME',
+    'PrefetchedBuilderFactory',
     'SlaveScanner',
     ]
 
@@ -705,9 +706,6 @@ class BuilddManager(service.Service):
     # How often to flush logtail updates, in seconds.
     FLUSH_LOGTAILS_INTERVAL = 15
 
-    # How often to update stats, in seconds
-    UPDATE_STATS_INTERVAL = 60
-
     def __init__(self, clock=None, builder_factory=None):
         # Use the clock if provided, it's so that tests can
         # advance it.  Use the reactor by default.
@@ -739,52 +737,6 @@ class BuilddManager(service.Service):
         logger.setLevel(level)
         return logger
 
-    def _updateBuilderCounts(self):
-        """Update statsd with the builder statuses."""
-        self.logger.debug("Updating builder stats.")
-        counts_by_processor = {}
-        for builder in self.builder_factory.iterVitals():
-            if not builder.active:
-                continue
-            for processor_name in builder.processor_names:
-                counts = counts_by_processor.setdefault(
-                    "{},virtualized={}".format(
-                        processor_name,
-                        builder.virtualized),
-                    {'cleaning': 0, 'idle': 0, 'disabled': 0, 'building': 0})
-                if not builder.builderok:
-                    counts['disabled'] += 1
-                elif builder.clean_status == BuilderCleanStatus.CLEANING:
-                    counts['cleaning'] += 1
-                elif (builder.build_queue and
-                      builder.build_queue.status == BuildQueueStatus.RUNNING):
-                    counts['building'] += 1
-                elif builder.clean_status == BuilderCleanStatus.CLEAN:
-                    counts['idle'] += 1
-        for processor, counts in counts_by_processor.items():
-            for count_name, count_value in counts.items():
-                gauge_name = "builders.{},arch={}".format(
-                    count_name, processor)
-                self.logger.debug("{}: {}".format(gauge_name, count_value))
-                self.statsd_client.gauge(gauge_name, count_value)
-        self.logger.debug("Builder stats update complete.")
-
-    def _updateBuilderQueues(self):
-        """Update statsd with the build queue lengths."""
-        self.logger.debug("Updating build queue stats.")
-        queue_details = getUtility(IBuilderSet).getBuildQueueSizes()
-        for queue_type, contents in queue_details.items():
-            virt = queue_type == 'virt'
-            for arch, value in contents.items():
-                gauge_name = "buildqueue,virtualized={},arch={}".format(
-                    virt, arch)
-                self.statsd_client.gauge(gauge_name, value[0])
-        self.logger.debug("Build queue stats update complete.")
-
-    def updateStats(self):
-        self._updateBuilderCounts()
-        self._updateBuilderQueues()
-
     def checkForNewBuilders(self):
         """Add and return any new builders."""
         new_builders = set(
@@ -854,9 +806,6 @@ class BuilddManager(service.Service):
         # Schedule bulk flushes for build queue logtail updates.
         self.flush_logtails_loop, self.flush_logtails_deferred = (
             self._startLoop(self.FLUSH_LOGTAILS_INTERVAL, self.flushLogTails))
-        # Schedule stats updates.
-        self.stats_update_loop, self.stats_update_deferred = (
-            self._startLoop(self.UPDATE_STATS_INTERVAL, self.updateStats))
 
     def stopService(self):
         """Callback for when we need to shut down."""
@@ -865,11 +814,9 @@ class BuilddManager(service.Service):
         deferreds = [slave.stopping_deferred for slave in self.builder_slaves]
         deferreds.append(self.scan_builders_deferred)
         deferreds.append(self.flush_logtails_deferred)
-        deferreds.append(self.stats_update_deferred)
 
         self.flush_logtails_loop.stop()
         self.scan_builders_loop.stop()
-        self.stats_update_loop.stop()
         for slave in self.builder_slaves:
             slave.stopCycle()
 
diff --git a/lib/lp/buildmaster/tests/test_manager.py b/lib/lp/buildmaster/tests/test_manager.py
index c7c8028..d9bbf9e 100644
--- a/lib/lp/buildmaster/tests/test_manager.py
+++ b/lib/lp/buildmaster/tests/test_manager.py
@@ -14,10 +14,7 @@ import signal
 import time
 
 from six.moves import xmlrpc_client
-from testtools.matchers import (
-    Equals,
-    MatchesListwise,
-    )
+from testtools.matchers import Equals
 from testtools.testcase import ExpectedException
 from testtools.twistedsupport import AsynchronousDeferredRunTest
 import transaction
@@ -48,7 +45,6 @@ from lp.buildmaster.interfaces.builder import (
     IBuilderSet,
     )
 from lp.buildmaster.interfaces.buildqueue import IBuildQueueSet
-from lp.buildmaster.interfaces.processor import IProcessorSet
 from lp.buildmaster.manager import (
     BuilddManager,
     BUILDER_FAILURE_THRESHOLD,
@@ -75,10 +71,9 @@ from lp.buildmaster.tests.test_interactor import (
     MockBuilderFactory,
     )
 from lp.registry.interfaces.distribution import IDistributionSet
-from lp.services.compat import mock
 from lp.services.config import config
 from lp.services.log.logger import BufferLogger
-from lp.services.statsd.interfaces.statsd_client import IStatsdClient
+from lp.services.statsd.tests import StatsMixin
 from lp.soyuz.interfaces.binarypackagebuild import IBinaryPackageBuildSet
 from lp.soyuz.model.binarypackagebuildbehaviour import (
     BinaryPackageBuildBehaviour,
@@ -93,7 +88,6 @@ from lp.testing import (
 from lp.testing.dbuser import switch_dbuser
 from lp.testing.factory import LaunchpadObjectFactory
 from lp.testing.fakemethod import FakeMethod
-from lp.testing.fixture import ZopeUtilityFixture
 from lp.testing.layers import (
     LaunchpadScriptLayer,
     LaunchpadZopelessLayer,
@@ -103,16 +97,6 @@ from lp.testing.matchers import HasQueryCount
 from lp.testing.sampledata import BOB_THE_BUILDER_NAME
 
 
-class StatsMixin:
-
-    def setUpStats(self):
-        # Install a mock statsd client so we can assert against the call
-        # counts and args.
-        self.stats_client = mock.Mock()
-        self.useFixture(
-            ZopeUtilityFixture(self.stats_client, IStatsdClient))
-
-
 class TestSlaveScannerScan(StatsMixin, TestCaseWithFactory):
     """Tests `SlaveScanner.scan` method.
 
@@ -1240,22 +1224,6 @@ class TestBuilddManager(TestCase):
         clock.advance(advance)
         self.assertNotEqual(0, manager.flushLogTails.call_count)
 
-    def test_startService_adds_updateStats_loop(self):
-        # When startService is called, the manager will start up a
-        # updateStats loop.
-        self._stub_out_scheduleNextScanCycle()
-        clock = task.Clock()
-        manager = BuilddManager(clock=clock)
-
-        # Replace updateStats() with FakeMethod so we can see if it was
-        # called.
-        manager.updateStats = FakeMethod()
-
-        manager.startService()
-        advance = BuilddManager.UPDATE_STATS_INTERVAL + 1
-        clock.advance(advance)
-        self.assertNotEqual(0, manager.updateStats.call_count)
-
 
 class TestFailureAssessments(TestCaseWithFactory):
 
@@ -1645,94 +1613,3 @@ class TestBuilddManagerScript(TestCaseWithFactory):
         self.assertFalse(
             os.access(rotated_logfilepath, os.F_OK),
             "Twistd's log file was rotated by twistd.")
-
-
-class TestStats(StatsMixin, TestCaseWithFactory):
-
-    layer = ZopelessDatabaseLayer
-    run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=20)
-
-    def setUp(self):
-        super(TestStats, self).setUp()
-        self.setUpStats()
-
-    def test_single_processor_counts(self):
-        builder = self.factory.makeBuilder()
-        builder.setCleanStatus(BuilderCleanStatus.CLEAN)
-        self.patch(BuilderSlave, 'makeBuilderSlave', FakeMethod(OkSlave()))
-        transaction.commit()
-        clock = task.Clock()
-        manager = BuilddManager(clock=clock)
-        manager._updateBuilderQueues = FakeMethod()
-        manager.builder_factory.update()
-        manager.updateStats()
-
-        self.assertEqual(8, self.stats_client.gauge.call_count)
-        for call in self.stats_client.mock.gauge.call_args_list:
-            self.assertIn('386', call[0][0])
-
-    def test_multiple_processor_counts(self):
-        builder = self.factory.makeBuilder(
-            processors=[getUtility(IProcessorSet).getByName('amd64')])
-        builder.setCleanStatus(BuilderCleanStatus.CLEAN)
-        self.patch(BuilderSlave, 'makeBuilderSlave', FakeMethod(OkSlave()))
-        transaction.commit()
-        clock = task.Clock()
-        manager = BuilddManager(clock=clock)
-        manager._updateBuilderQueues = FakeMethod()
-        manager.builder_factory.update()
-        manager.updateStats()
-
-        self.assertEqual(12, self.stats_client.gauge.call_count)
-        i386_calls = [c for c in self.stats_client.gauge.call_args_list
-                      if '386' in c[0][0]]
-        amd64_calls = [c for c in self.stats_client.gauge.call_args_list
-                       if 'amd64' in c[0][0]]
-        self.assertEqual(8, len(i386_calls))
-        self.assertEqual(4, len(amd64_calls))
-
-    def test_correct_values_counts(self):
-        builder = self.factory.makeBuilder(
-            processors=[getUtility(IProcessorSet).getByName('amd64')])
-        builder.setCleanStatus(BuilderCleanStatus.CLEANING)
-        self.patch(BuilderSlave, 'makeBuilderSlave', FakeMethod(OkSlave()))
-        transaction.commit()
-        clock = task.Clock()
-        manager = BuilddManager(clock=clock)
-        manager._updateBuilderQueues = FakeMethod()
-        manager.builder_factory.update()
-        manager.updateStats()
-
-        self.assertEqual(12, self.stats_client.gauge.call_count)
-        calls = [c[0] for c in self.stats_client.gauge.call_args_list
-                 if 'amd64' in c[0][0]]
-        self.assertThat(
-            calls, MatchesListwise(
-                [Equals(('builders.disabled,arch=amd64,virtualized=True', 0)),
-                 Equals(('builders.building,arch=amd64,virtualized=True', 0)),
-                 Equals(('builders.idle,arch=amd64,virtualized=True', 0)),
-                 Equals(('builders.cleaning,arch=amd64,virtualized=True', 1))
-                 ]))
-
-    def test_updateBuilderQueues(self):
-        builder = self.factory.makeBuilder(
-            processors=[getUtility(IProcessorSet).getByName('amd64')])
-        builder.setCleanStatus(BuilderCleanStatus.CLEANING)
-        build = self.factory.makeSnapBuild()
-        build.queueBuild()
-        self.patch(BuilderSlave, 'makeBuilderSlave', FakeMethod(OkSlave()))
-        transaction.commit()
-        clock = task.Clock()
-        manager = BuilddManager(clock=clock)
-        manager._updateBuilderCounts = FakeMethod()
-        manager.builder_factory.update()
-        manager.updateStats()
-
-        self.assertEqual(2, self.stats_client.gauge.call_count)
-        self.assertThat(
-            [x[0] for x in self.stats_client.gauge.call_args_list],
-            MatchesListwise(
-                [Equals(('buildqueue,virtualized=True,arch={}'.format(
-                    build.processor.name), 1)),
-                 Equals(('buildqueue,virtualized=False,arch=386', 1))
-                 ]))
diff --git a/lib/lp/services/statsd/numbercruncher.py b/lib/lp/services/statsd/numbercruncher.py
new file mode 100644
index 0000000..0798719
--- /dev/null
+++ b/lib/lp/services/statsd/numbercruncher.py
@@ -0,0 +1,137 @@
+# Copyright 2020 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Out of process statsd reporting."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = ['NumberCruncher']
+
+import logging
+
+from twisted.application import service
+from twisted.internet import (
+    defer,
+    reactor,
+    )
+from twisted.internet.task import LoopingCall
+from twisted.python import log
+from zope.component import getUtility
+
+from lp.buildmaster.enums import (
+    BuilderCleanStatus,
+    BuildQueueStatus,
+    )
+from lp.buildmaster.interfaces.builder import IBuilderSet
+from lp.buildmaster.manager import PrefetchedBuilderFactory
+from lp.services.statsd.interfaces.statsd_client import IStatsdClient
+
+NUMBER_CRUNCHER_LOG_NAME = "number-cruncher"
+
+
+class NumberCruncher(service.Service):
+    """Export statistics to statsd."""
+
+    QUEUE_INTERVAL = 60
+    BUILDER_INTERVAL = 60
+
+    def __init__(self, clock=None, builder_factory=None):
+        if clock is None:
+            clock = reactor
+        self._clock = clock
+        self.logger = self._setupLogger()
+        self.builder_factory = builder_factory or PrefetchedBuilderFactory()
+        self.statsd_client = getUtility(IStatsdClient)
+
+    def _setupLogger(self):
+        """Set up a 'number-cruncher' logger that redirects to twisted.
+        """
+        level = logging.INFO
+        logger = logging.getLogger(NUMBER_CRUNCHER_LOG_NAME)
+        logger.propagate = False
+
+        # Redirect the output to the twisted log module.
+        channel = logging.StreamHandler(log.StdioOnnaStick())
+        channel.setLevel(level)
+        channel.setFormatter(logging.Formatter('%(message)s'))
+
+        logger.addHandler(channel)
+        logger.setLevel(level)
+        return logger
+
+    def _startLoop(self, interval, callback):
+        """Schedule `callback` to run every `interval` seconds."""
+        loop = LoopingCall(callback)
+        loop.clock = self._clock
+        stopping_deferred = loop.start(interval)
+        return loop, stopping_deferred
+
+    def updateBuilderQueues(self):
+        """Update statsd with the build queue lengths."""
+        self.logger.debug("Updating build queue stats.")
+        queue_details = getUtility(IBuilderSet).getBuildQueueSizes()
+        for queue_type, contents in queue_details.items():
+            virt = queue_type == 'virt'
+            for arch, value in contents.items():
+                gauge_name = "buildqueue,virtualized={},arch={}".format(
+                    virt, arch)
+                self.logger.debug("{}: {}".format(gauge_name, value[0]))
+                self.statsd_client.gauge(gauge_name, value[0])
+        self.logger.debug("Build queue stats update complete.")
+
+    def _updateBuilderCounts(self):
+        """Update statsd with the builder statuses.
+
+        Requires the builder_factory to be updated.
+        """
+        self.logger.debug("Updating builder stats.")
+        counts_by_processor = {}
+        for builder in self.builder_factory.iterVitals():
+            if not builder.active:
+                continue
+            for processor_name in builder.processor_names:
+                counts = counts_by_processor.setdefault(
+                    "{},virtualized={}".format(
+                        processor_name,
+                        builder.virtualized),
+                    {'cleaning': 0, 'idle': 0, 'disabled': 0, 'building': 0})
+                if not builder.builderok:
+                    counts['disabled'] += 1
+                elif builder.clean_status == BuilderCleanStatus.CLEANING:
+                    counts['cleaning'] += 1
+                elif (builder.build_queue and
+                      builder.build_queue.status == BuildQueueStatus.RUNNING):
+                    counts['building'] += 1
+                elif builder.clean_status == BuilderCleanStatus.CLEAN:
+                    counts['idle'] += 1
+        for processor, counts in counts_by_processor.items():
+            for count_name, count_value in counts.items():
+                gauge_name = "builders.{},arch={}".format(
+                    count_name, processor)
+                self.logger.debug("{}: {}".format(gauge_name, count_value))
+                self.statsd_client.gauge(gauge_name, count_value)
+        self.logger.debug("Builder stats update complete.")
+
+    def updateBuilderStats(self):
+        """Statistics that require builder knowledge to be updated."""
+        self.builder_factory.update()
+        self._updateBuilderCounts()
+
+    def startService(self):
+        self.logger.INFO("Starting number-cruncher service.")
+        self.update_queues_loop, self.update_queues_deferred = (
+            self._startLoop(self.QUEUE_INTERVAL, self.updateBuilderQueues))
+        self.update_builder_loop, self.update_builder_deferred = (
+            self._startLoop(self.BUILDER_INTERVAL, self.updateBuilderStats))
+
+    def stopService(self):
+        deferreds = []
+        deferreds.append(self.update_queues_deferred)
+        deferreds.append(self.update_builder_deferred)
+
+        self.update_queues_loop.stop()
+        self.update_builder_loop.stop()
+
+        d = defer.DeferredList(deferreds, consumeErrors=True)
+        return d
diff --git a/lib/lp/services/statsd/tests/__init__.py b/lib/lp/services/statsd/tests/__init__.py
index e69de29..20885a4 100644
--- a/lib/lp/services/statsd/tests/__init__.py
+++ b/lib/lp/services/statsd/tests/__init__.py
@@ -0,0 +1,23 @@
+# Copyright 2020 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Utility mixins for testing statsd handling"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = ['StatsMixin']
+
+from lp.services.compat import mock
+from lp.services.statsd.interfaces.statsd_client import IStatsdClient
+from lp.testing.fixture import ZopeUtilityFixture
+
+
+class StatsMixin:
+
+    def setUpStats(self):
+        # Install a mock statsd client so we can assert against the call
+        # counts and args.
+        self.stats_client = mock.Mock()
+        self.useFixture(
+            ZopeUtilityFixture(self.stats_client, IStatsdClient))
diff --git a/lib/lp/services/statsd/tests/test_numbercruncher.py b/lib/lp/services/statsd/tests/test_numbercruncher.py
new file mode 100644
index 0000000..d812037
--- /dev/null
+++ b/lib/lp/services/statsd/tests/test_numbercruncher.py
@@ -0,0 +1,115 @@
+# Copyright 2020 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Tests for the stats number cruncher daemon."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+
+from testtools.matchers import (
+    Equals,
+    MatchesListwise,
+    )
+from testtools.twistedsupport import AsynchronousDeferredRunTest
+import transaction
+from twisted.internet import task
+from zope.component import getUtility
+
+from lp.buildmaster.enums import BuilderCleanStatus
+from lp.buildmaster.interactor import BuilderSlave
+from lp.buildmaster.interfaces.processor import IProcessorSet
+from lp.buildmaster.tests.mock_slaves import OkSlave
+from lp.services.statsd.numbercruncher import NumberCruncher
+from lp.services.statsd.tests import StatsMixin
+from lp.testing import TestCaseWithFactory
+from lp.testing.fakemethod import FakeMethod
+from lp.testing.layers import ZopelessDatabaseLayer
+
+
+class TestNumberCruncher(StatsMixin, TestCaseWithFactory):
+
+    layer = ZopelessDatabaseLayer
+    run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=20)
+
+    def setUp(self):
+        super(TestNumberCruncher, self).setUp()
+        self.setUpStats()
+
+    def test_single_processor_counts(self):
+        builder = self.factory.makeBuilder()
+        builder.setCleanStatus(BuilderCleanStatus.CLEAN)
+        self.patch(BuilderSlave, 'makeBuilderSlave', FakeMethod(OkSlave()))
+        transaction.commit()
+        clock = task.Clock()
+        manager = NumberCruncher(clock=clock)
+        manager.builder_factory.update()
+        manager.updateBuilderStats()
+
+        self.assertEqual(8, self.stats_client.gauge.call_count)
+        for call in self.stats_client.mock.gauge.call_args_list:
+            self.assertIn('386', call[0][0])
+
+    def test_multiple_processor_counts(self):
+        builder = self.factory.makeBuilder(
+            processors=[getUtility(IProcessorSet).getByName('amd64')])
+        builder.setCleanStatus(BuilderCleanStatus.CLEAN)
+        self.patch(BuilderSlave, 'makeBuilderSlave', FakeMethod(OkSlave()))
+        transaction.commit()
+        clock = task.Clock()
+        manager = NumberCruncher(clock=clock)
+        manager.builder_factory.update()
+        manager.updateBuilderStats()
+
+        self.assertEqual(12, self.stats_client.gauge.call_count)
+        i386_calls = [c for c in self.stats_client.gauge.call_args_list
+                      if '386' in c[0][0]]
+        amd64_calls = [c for c in self.stats_client.gauge.call_args_list
+                       if 'amd64' in c[0][0]]
+        self.assertEqual(8, len(i386_calls))
+        self.assertEqual(4, len(amd64_calls))
+
+    def test_correct_values_counts(self):
+        builder = self.factory.makeBuilder(
+            processors=[getUtility(IProcessorSet).getByName('amd64')])
+        builder.setCleanStatus(BuilderCleanStatus.CLEANING)
+        self.patch(BuilderSlave, 'makeBuilderSlave', FakeMethod(OkSlave()))
+        transaction.commit()
+        clock = task.Clock()
+        manager = NumberCruncher(clock=clock)
+        manager.builder_factory.update()
+        manager.updateBuilderStats()
+
+        self.assertEqual(12, self.stats_client.gauge.call_count)
+        calls = [c[0] for c in self.stats_client.gauge.call_args_list
+                 if 'amd64' in c[0][0]]
+        self.assertThat(
+            calls, MatchesListwise(
+                [Equals(('builders.disabled,arch=amd64,virtualized=True', 0)),
+                 Equals(('builders.building,arch=amd64,virtualized=True', 0)),
+                 Equals(('builders.idle,arch=amd64,virtualized=True', 0)),
+                 Equals(('builders.cleaning,arch=amd64,virtualized=True', 1))
+                 ]))
+
+    def test_updateBuilderQueues(self):
+        builder = self.factory.makeBuilder(
+            processors=[getUtility(IProcessorSet).getByName('amd64')])
+        builder.setCleanStatus(BuilderCleanStatus.CLEANING)
+        build = self.factory.makeSnapBuild()
+        build.queueBuild()
+        self.patch(BuilderSlave, 'makeBuilderSlave', FakeMethod(OkSlave()))
+        transaction.commit()
+        clock = task.Clock()
+        manager = NumberCruncher(clock=clock)
+        manager._updateBuilderCounts = FakeMethod()
+        manager.builder_factory.update()
+        manager.updateBuilderQueues()
+
+        self.assertEqual(2, self.stats_client.gauge.call_count)
+        self.assertThat(
+            [x[0] for x in self.stats_client.gauge.call_args_list],
+            MatchesListwise(
+                [Equals(('buildqueue,virtualized=True,arch={}'.format(
+                    build.processor.name), 1)),
+                 Equals(('buildqueue,virtualized=False,arch=386', 1))
+                 ]))
diff --git a/utilities/start-dev-soyuz.sh b/utilities/start-dev-soyuz.sh
index c151c1c..efe4b0f 100755
--- a/utilities/start-dev-soyuz.sh
+++ b/utilities/start-dev-soyuz.sh
@@ -28,6 +28,7 @@ start_twistd_plugin() {
 
 start_twistd testkeyserver lib/lp/testing/keyserver/testkeyserver.tac
 start_twistd buildd-manager daemons/buildd-manager.tac
+start_twistd numbercruncher daemons/numbercruncher.tac
 mkdir -p /var/tmp/txpkgupload/incoming
 start_twistd_plugin txpkgupload pkgupload \
     --config-file configs/development/txpkgupload.yaml