Merge ~andreserl/maas:prometheus_minor_reorg into maas:master

[Andres Rodriguez]

Minor code re-org and addition of statuses to later use as a base for a /metrics endpoint.

This is a re-org which was originally part of https://code.launchpad.net/~andreserl/maas/+git/maas/+merge/356087 but decided to do this so I can start adding more stats without having later conflicts.

62aa331... by Andres Rodriguez on 2018-10-15

Improvement

[Lee Trager (ltrager)]

LGTM!

review: Approve

diff --git a/src/maasserver/prometheus.py b/src/maasserver/prometheus.py
index f736de9..d30aad3 100644
--- a/src/maasserver/prometheus.py
+++ b/src/maasserver/prometheus.py
@@ -32,7 +32,7 @@ except:
 log = LegacyLogger()
 
 
-def push_stats_to_prometheus(maas_name, push_gateway):
+def get_stats_for_prometheus():
     registry = CollectorRegistry()
     stats = json.loads(get_maas_stats())
 
@@ -43,6 +43,11 @@ def push_stats_to_prometheus(maas_name, push_gateway):
     for status, machines in stats['machine_status'].items():
         counter.labels(status).set(machines)
 
+    return registry
+
+
+def push_stats_to_prometheus(maas_name, push_gateway):
+    registry = get_stats_for_prometheus()
     push_to_gateway(
         push_gateway, job='stats_for_%s' % maas_name, registry=registry)
 
diff --git a/src/maasserver/stats.py b/src/maasserver/stats.py
index d0544b9..5f43bdc 100644
--- a/src/maasserver/stats.py
+++ b/src/maasserver/stats.py
@@ -53,14 +53,23 @@ def get_machine_state_stats():
     node_status = Counter(node_status)
 
     return {
+        # base status
+        "new": node_status.get(NODE_STATUS.NEW, 0),
         "ready": node_status.get(NODE_STATUS.READY, 0),
         "allocated": node_status.get(NODE_STATUS.ALLOCATED, 0),
-        "deploying": node_status.get(NODE_STATUS.DEPLOYING, 0),
         "deployed": node_status.get(NODE_STATUS.DEPLOYED, 0),
+        # in progress status
+        "commissioning": node_status.get(NODE_STATUS.COMMISSIONING, 0),
+        "testing": node_status.get(NODE_STATUS.TESTING, 0),
+        "deploying": node_status.get(NODE_STATUS.DEPLOYING, 0),
+        # failure status
         "failed_deployment": node_status.get(
             NODE_STATUS.FAILED_DEPLOYMENT, 0),
         "failed_commissioning": node_status.get(
             NODE_STATUS.FAILED_COMMISSIONING, 0),
+        "failed_testing": node_status.get(
+            NODE_STATUS.FAILED_TESTING, 0),
+        "broken": node_status.get(NODE_STATUS.BROKEN, 0),
         }
 
 
@@ -102,9 +111,9 @@ def get_maas_stats():
             "machines": node_types.get(NODE_TYPE.MACHINE, 0),
             "devices": node_types.get(NODE_TYPE.DEVICE, 0),
         },
-        "machine_stats": stats,
-        "machine_status": machine_status,
-        "network_stats": netstats,
+        "machine_stats": stats,  # count of cpus, mem, storage
+        "machine_status": machine_status,  # machines by status
+        "network_stats": netstats,  # network status
     })
 
 
diff --git a/src/maasserver/tests/test_prometheus.py b/src/maasserver/tests/test_prometheus.py
index 27dc7f4..66c1af6 100644
--- a/src/maasserver/tests/test_prometheus.py
+++ b/src/maasserver/tests/test_prometheus.py
@@ -5,10 +5,15 @@
 
 __all__ = []
 
+import json
+
 from django.db import transaction
 from maasserver import prometheus
 from maasserver.models import Config
-from maasserver.prometheus import push_stats_to_prometheus
+from maasserver.prometheus import (
+    get_stats_for_prometheus,
+    push_stats_to_prometheus,
+)
 from maasserver.testing.factory import factory
 from maasserver.testing.testcase import (
     MAASServerTestCase,
@@ -28,6 +33,20 @@ from twisted.internet.defer import fail
 
 class TestPrometheus(MAASServerTestCase):
 
+    def test_get_stats_for_prometheus(self):
+        self.patch(prometheus, "CollectorRegistry")
+        self.patch(prometheus, "Gauge")
+        values = {
+            "machine_status": {
+                "random_status": 0,
+            },
+        }
+        mock = self.patch(prometheus, "get_maas_stats")
+        mock.return_value = json.dumps(values)
+        get_stats_for_prometheus()
+        self.assertThat(
+            mock, MockCalledOnce())
+
     def test_push_stats_to_prometheus(self):
         factory.make_RegionRackController()
         maas_name = 'random.maas'
diff --git a/src/maasserver/tests/test_stats.py b/src/maasserver/tests/test_stats.py
index ee7a657..7f27ba0 100644
--- a/src/maasserver/tests/test_stats.py
+++ b/src/maasserver/tests/test_stats.py
@@ -6,13 +6,16 @@
 __all__ = []
 
 import base64
+from collections import Counter
 import json
 
 from django.db import transaction
 from maasserver import stats
+from maasserver.enum import NODE_STATUS
 from maasserver.models import (
     Config,
     Fabric,
+    Node,
     Space,
     Subnet,
     VLAN,
@@ -63,6 +66,9 @@ class TestMAASStats(MAASServerTestCase):
         # calculates, so just get it directly from the database for the test.
         total_storage = machine_stats['total_storage']
 
+        node_status = Node.objects.values_list('status', flat=True)
+        node_status = Counter(node_status)
+
         compare = {
             "controllers": {
                 "regionracks": 1,
@@ -79,12 +85,23 @@ class TestMAASStats(MAASServerTestCase):
                 "total_storage": total_storage,
             },
             "machine_status": {
-                "ready": 1,
-                "allocated": 0,
-                "deploying": 0,
-                "deployed": 0,
-                "failed_deployment": 1,
-                "failed_commissioning": 0,
+                "new": node_status.get(NODE_STATUS.NEW, 0),
+                "ready": node_status.get(NODE_STATUS.READY, 0),
+                "allocated": node_status.get(NODE_STATUS.ALLOCATED, 0),
+                "deployed": node_status.get(NODE_STATUS.DEPLOYED, 0),
+                "commissioning": node_status.get(
+                    NODE_STATUS.COMMISSIONING, 0),
+                "testing": node_status.get(
+                    NODE_STATUS.TESTING, 0),
+                "deploying": node_status.get(
+                    NODE_STATUS.DEPLOYING, 0),
+                "failed_deployment": node_status.get(
+                    NODE_STATUS.FAILED_DEPLOYMENT, 0),
+                "failed_commissioning": node_status.get(
+                    NODE_STATUS.COMMISSIONING, 0),
+                "failed_testing": node_status.get(
+                    NODE_STATUS.FAILED_TESTING, 0),
+                "broken": node_status.get(NODE_STATUS.BROKEN, 0),
             },
             "network_stats": {
                 "spaces": Space.objects.count(),