Merge ~andreserl/maas:2.5_prometheus_integration into maas:master

[Andres Rodriguez]

Add initial integration to publish stats to prometheus.

[Blake Rouse (blake-rouse)]

Its a start, but has a lot of issues with usage of the database at the wrong times. Those must be fixed before this can land.

review: Needs Fixing

[Blake Rouse (blake-rouse)]

Looks good.

You do have one issues. The interval will not be updated automatically, it will only be updated one the old interval is reached. So on startup it will work fine, but changing it to 30 minutes, will not update until 60 minutes is reached. Then it will update from that point.

[Andres Rodriguez (andreserl)]

Question inlien!

[Blake Rouse (blake-rouse)]

Looks good, just one comment on the testing.

review: Approve

[MAAS Lander (maas-lander)]

LANDING
-b 2.5_prometheus_integration lp:~andreserl/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED BUILD
LOG: http://maas-ci-jenkins.internal:8080/job/maas/job/branch-tester/4048/consoleText

[MAAS Lander (maas-lander)]

LANDING
-b 2.5_prometheus_integration lp:~andreserl/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED BUILD
LOG: http://maas-ci-jenkins.internal:8080/job/maas/job/branch-tester/4051/consoleText

[MAAS Lander (maas-lander)]

LANDING
-b 2.5_prometheus_integration lp:~andreserl/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED BUILD
LOG: http://maas-ci-jenkins.internal:8080/job/maas/job/branch-tester/4052/consoleText

[MAAS Lander (maas-lander)]

LANDING
-b 2.5_prometheus_integration lp:~andreserl/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED BUILD
LOG: http://maas-ci-jenkins.internal:8080/job/maas/job/branch-tester/4054/consoleText

0e9c7f2... by Andres Rodriguez on 2018-10-03

Make format

diff --git a/src/maasserver/eventloop.py b/src/maasserver/eventloop.py
index 3c71695..af4f10e 100644
--- a/src/maasserver/eventloop.py
+++ b/src/maasserver/eventloop.py
@@ -104,6 +104,11 @@ def make_StatsService():
     return stats.StatsService()
 
 
+def make_PrometheusService():
+    from maasserver import prometheus
+    return prometheus.PrometheusService()
+
+
 def make_ImportResourcesService():
     from maasserver import bootresources
     return bootresources.ImportResourcesService()
@@ -259,6 +264,11 @@ class RegionEventLoop:
             "factory": make_StatsService,
             "requires": [],
         },
+        "prometheus": {
+            "only_on_master": True,
+            "factory": make_PrometheusService,
+            "requires": [],
+        },
         "import-resources": {
             "only_on_master": False,
             "import_service": True,
diff --git a/src/maasserver/forms/settings.py b/src/maasserver/forms/settings.py
index 730eba4..0b91e06 100644
--- a/src/maasserver/forms/settings.py
+++ b/src/maasserver/forms/settings.py
@@ -729,6 +729,43 @@ CONFIG_ITEMS = {
             'min_value': 1,
         },
     },
+    'prometheus_enabled': {
+        'default': False,
+        'form': forms.BooleanField,
+        'form_kwargs': {
+            'label': (
+                "Enable sending stats to a prometheus gateway."),
+            'required': False,
+            'help_text': (
+                "Allows MAAS to send statistics to Prometheus. This requires "
+                "the 'prometheus_push_gateway' to be set.")
+        }
+    },
+    'prometheus_push_gateway': {
+        'default': None,
+        'form': forms.CharField,
+        'form_kwargs': {
+            'label': (
+                "Address or hostname of the Prometheus push gateway."),
+            'required': False,
+            'help_text': (
+                "Defines the address or hostname of the Prometheus push "
+                "gateway where MAAS will send data to.")
+        }
+    },
+    'prometheus_push_interval': {
+        'default': 60,
+        'form': forms.IntegerField,
+        'form_kwargs': {
+            'label': (
+                "Interval of how often to send data to Prometheus "
+                "(default: to 60 minutes)."),
+            'required': False,
+            'help_text': (
+                "The internal of how often MAAS will send stats to Prometheus "
+                "in minutes.")
+        }
+    },
 }
 
 
diff --git a/src/maasserver/models/config.py b/src/maasserver/models/config.py
index c7a8cbb..59e7c46 100644
--- a/src/maasserver/models/config.py
+++ b/src/maasserver/models/config.py
@@ -125,6 +125,10 @@ def get_default_config():
         # MAAS Architecture.
         'use_rack_proxy': True,
         'node_timeout': 30,
+        # prometheus.
+        'prometheus_enabled': False,
+        'prometheus_push_gateway': None,
+        'prometheus_push_interval': 60,
     }
 
 
diff --git a/src/maasserver/prometheus.py b/src/maasserver/prometheus.py
 new file mode 100644
index 0000000..f736de9
--- /dev/null
+++ b/src/maasserver/prometheus.py
@@ -0,0 +1,96 @@
+# Copyright 2017 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Prometheus integration"""
+
+__all__ = [
+    "PrometheusService",
+    "PROMETHEUS_SERVICE_PERIOD",
+]
+
+from datetime import timedelta
+import json
+
+from maasserver.models import Config
+from maasserver.stats import get_maas_stats
+from maasserver.utils.orm import transactional
+from maasserver.utils.threads import deferToDatabase
+from provisioningserver.logger import LegacyLogger
+from twisted.application.internet import TimerService
+
+
+try:
+    from prometheus_client import (
+        CollectorRegistry,
+        Gauge,
+        push_to_gateway,
+    )
+    PROMETHEUS = True
+except:
+    PROMETHEUS = False
+
+log = LegacyLogger()
+
+
+def push_stats_to_prometheus(maas_name, push_gateway):
+    registry = CollectorRegistry()
+    stats = json.loads(get_maas_stats())
+
+    # Gather counter for machines per status
+    counter = Gauge(
+        "machine_status", "Number per machines per stats",
+        ["status"], registry=registry)
+    for status, machines in stats['machine_status'].items():
+        counter.labels(status).set(machines)
+
+    push_to_gateway(
+        push_gateway, job='stats_for_%s' % maas_name, registry=registry)
+
+
+# Define the default time the service interval is run.
+# This can be overriden by the config option.
+PROMETHEUS_SERVICE_PERIOD = timedelta(minutes=60)
+
+
+class PrometheusService(TimerService, object):
+    """Service to periodically push stats to Prometheus
+
+    This will run immediately when it's started, by default, it will run
+    every 60 minutes, though the interval can be overridden (see
+    prometheus_push_internval global config).
+    """
+
+    def __init__(self, interval=PROMETHEUS_SERVICE_PERIOD):
+        super(PrometheusService, self).__init__(
+            interval.total_seconds(), self.maybe_push_prometheus_stats)
+
+    def maybe_push_prometheus_stats(self):
+        def determine_stats_request():
+            config = Config.objects.get_configs([
+                'maas_name', 'prometheus_enabled', 'prometheus_push_gateway',
+                'prometheus_push_interval'])
+            # Update interval
+            self._update_interval(
+                timedelta(minutes=config['prometheus_push_interval']))
+            # Determine if we can run the actual update.
+            if (not PROMETHEUS or not config['prometheus_enabled'] or
+                    config['prometheus_push_gateway'] is None):
+                return
+            # Run updates.
+            push_stats_to_prometheus(
+                config['maas_name'], config['prometheus_push_gateway'])
+
+        d = deferToDatabase(transactional(determine_stats_request))
+        d.addErrback(
+            log.err,
+            "Failure pushing stats to prometheus gateway")
+        return d
+
+    def _update_interval(self, interval):
+        """Change the update interval."""
+        interval_seconds = interval.total_seconds()
+        if self.step == interval_seconds:
+            return
+        self._loop.interval = self.step = interval_seconds
+        if self._loop.running:
+            self._loop.reset()
diff --git a/src/maasserver/tests/test_eventloop.py b/src/maasserver/tests/test_eventloop.py
index aecb077..4a6f7c0 100644
--- a/src/maasserver/tests/test_eventloop.py
+++ b/src/maasserver/tests/test_eventloop.py
@@ -18,6 +18,7 @@ from maasserver import (
     eventloop,
     ipc,
     nonces_cleanup,
+    prometheus,
     rack_controller,
     region_controller,
     stats,
@@ -376,6 +377,17 @@ class TestFactories(MAASTestCase):
         self.assertTrue(
             eventloop.loop.factories["stats"]["only_on_master"])
 
+    def test_make_PrometheusService(self):
+        service = eventloop.make_PrometheusService()
+        self.assertThat(service, IsInstance(
+            prometheus.PrometheusService))
+        # It is registered as a factory in RegionEventLoop.
+        self.assertIs(
+            eventloop.make_PrometheusService,
+            eventloop.loop.factories["prometheus"]["factory"])
+        self.assertTrue(
+            eventloop.loop.factories["prometheus"]["only_on_master"])
+
     def test_make_ImportResourcesService(self):
         service = eventloop.make_ImportResourcesService()
         self.assertThat(service, IsInstance(
diff --git a/src/maasserver/tests/test_plugin.py b/src/maasserver/tests/test_plugin.py
index 1ee6898..d3b04fe 100644
--- a/src/maasserver/tests/test_plugin.py
+++ b/src/maasserver/tests/test_plugin.py
@@ -228,6 +228,7 @@ class TestRegionMasterServiceMaker(TestServiceMaker):
             "service-monitor",
             "status-monitor",
             "stats",
+            "prometheus",
             "postgres-listener-master",
             "networks-monitor",
             "active-discovery",
@@ -322,6 +323,7 @@ class TestRegionAllInOneServiceMaker(TestServiceMaker):
             "dns-publication-cleanup",
             "status-monitor",
             "stats",
+            "prometheus",
             "import-resources",
             "import-resources-progress",
             "postgres-listener-master",
diff --git a/src/maasserver/tests/test_prometheus.py b/src/maasserver/tests/test_prometheus.py
 new file mode 100644
index 0000000..27dc7f4
--- /dev/null
+++ b/src/maasserver/tests/test_prometheus.py
@@ -0,0 +1,103 @@
+# Copyright 2014-2018 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Test maasserver.prometheus."""
+
+__all__ = []
+
+from django.db import transaction
+from maasserver import prometheus
+from maasserver.models import Config
+from maasserver.prometheus import push_stats_to_prometheus
+from maasserver.testing.factory import factory
+from maasserver.testing.testcase import (
+    MAASServerTestCase,
+    MAASTransactionServerTestCase,
+)
+from maastesting.matchers import (
+    MockCalledOnce,
+    MockCalledOnceWith,
+    MockNotCalled,
+)
+from maastesting.testcase import MAASTestCase
+from maastesting.twisted import extract_result
+from provisioningserver.utils.twisted import asynchronous
+from twisted.application.internet import TimerService
+from twisted.internet.defer import fail
+
+
+class TestPrometheus(MAASServerTestCase):
+
+    def test_push_stats_to_prometheus(self):
+        factory.make_RegionRackController()
+        maas_name = 'random.maas'
+        push_gateway = '127.0.0.1:2000'
+        registry_mock = self.patch(prometheus, "CollectorRegistry")
+        self.patch(prometheus, "Gauge")
+        mock = self.patch(prometheus, "push_to_gateway")
+        push_stats_to_prometheus(maas_name, push_gateway)
+        self.assertThat(
+            mock, MockCalledOnceWith(
+                push_gateway,
+                job="stats_for_%s" % maas_name,
+                registry=registry_mock()))
+
+
+class TestPrometheusService(MAASTestCase):
+    """Tests for `ImportPrometheusService`."""
+
+    def test__is_a_TimerService(self):
+        service = prometheus.PrometheusService()
+        self.assertIsInstance(service, TimerService)
+
+    def test__runs_once_an_hour_by_default(self):
+        service = prometheus.PrometheusService()
+        self.assertEqual(3600, service.step)
+
+    def test__calls__maybe_make_stats_request(self):
+        service = prometheus.PrometheusService()
+        self.assertEqual(
+            (service.maybe_push_prometheus_stats, (), {}),
+            service.call)
+
+    def test_maybe_make_stats_request_does_not_error(self):
+        service = prometheus.PrometheusService()
+        deferToDatabase = self.patch(prometheus, "deferToDatabase")
+        exception_type = factory.make_exception_type()
+        deferToDatabase.return_value = fail(exception_type())
+        d = service.maybe_push_prometheus_stats()
+        self.assertIsNone(extract_result(d))
+
+
+class TestPrometheusServiceAsync(MAASTransactionServerTestCase):
+    """Tests for the async parts of `PrometheusService`."""
+
+    def test_maybe_make_stats_request_makes_request(self):
+        mock_call = self.patch(prometheus, "push_stats_to_prometheus")
+        setting = self.patch(prometheus, "PROMETHEUS")
+        setting.return_value = True
+
+        with transaction.atomic():
+            Config.objects.set_config('prometheus_enabled', True)
+            Config.objects.set_config(
+                'prometheus_push_gateway', '192.168.1.1:8081')
+
+        service = prometheus.PrometheusService()
+        maybe_push_prometheus_stats = asynchronous(
+            service.maybe_push_prometheus_stats)
+        maybe_push_prometheus_stats().wait(5)
+
+        self.assertThat(mock_call, MockCalledOnce())
+
+    def test_maybe_make_stats_request_doesnt_make_request(self):
+        mock_call = self.patch(prometheus, "push_stats_to_prometheus")
+
+        with transaction.atomic():
+            Config.objects.set_config('enable_analytics', False)
+
+        service = prometheus.PrometheusService()
+        maybe_push_prometheus_stats = asynchronous(
+            service.maybe_push_prometheus_stats)
+        maybe_push_prometheus_stats().wait(5)
+
+        self.assertThat(mock_call, MockNotCalled())
diff --git a/utilities/check-imports b/utilities/check-imports
index 69c445c..9a8a90f 100755
--- a/utilities/check-imports
+++ b/utilities/check-imports
@@ -273,6 +273,7 @@ RegionControllerRule = Rule(
     Allow("OpenSSL|OpenSSL.**"),
     Allow("petname"),
     Allow("piston3|piston3.**"),
+    Allow("prometheus_client.**"),
     Allow("provisioningserver|provisioningserver.**"),
     Allow("psycopg2|psycopg2.**"),
     Allow("pytz.UTC"),