Merge ~ack/maas:fix-prometheus-stats into maas:master

[Alberto Donato]

LP: #1813281 - coalesce counts in stats to prevent null values

[MAAS Lander (maas-lander)]

UNIT TESTS
-b fix-prometheus-stats lp:~ack/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED
LOG: http://maas-ci-jenkins.internal:8080/job/maas/job/branch-tester/4955/console
COMMIT: 8a474263807ff87966f76a75e29e3415b2bc49cc

review: Needs Fixing

[MAAS Lander (maas-lander)]

UNIT TESTS
-b fix-prometheus-stats lp:~ack/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED
LOG: http://maas-ci-jenkins.internal:8080/job/maas/job/branch-tester/4960/console
COMMIT: efe31469c7b192bbe614461b90d5e4a6344cb493

review: Needs Fixing

[Björn Tillenius (bjornt)]

+1

review: Approve

[MAAS Lander (maas-lander)]

LANDING
-b fix-prometheus-stats lp:~ack/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED BUILD
LOG: http://maas-ci-jenkins.internal:8080/job/maas/job/branch-tester/4975/consoleText

diff --git a/src/maasserver/stats.py b/src/maasserver/stats.py
index d0d7f1c..8b55ae9 100644
--- a/src/maasserver/stats.py
+++ b/src/maasserver/stats.py
@@ -10,7 +10,11 @@ __all__ = [
 
 from datetime import timedelta
 
-from django.db.models import Sum
+from django.db.models import (
+    Sum,
+    Value,
+)
+from django.db.models.functions import Coalesce
 from maasserver.models import Config
 from maasserver.utils.orm import transactional
 from maasserver.utils.threads import deferToDatabase
@@ -21,10 +25,7 @@ from twisted.application.internet import TimerService
 log = LegacyLogger()
 
 import base64
-from collections import (
-    Counter,
-    defaultdict,
-)
+from collections import Counter
 import json
 
 from maasserver.enum import (
@@ -45,13 +46,19 @@ from maasserver.utils import get_maas_user_agent
 import requests
 
 
+def NotNullSum(column):
+    """Like Sum, but returns 0 if the aggregate is None."""
+    return Coalesce(Sum(column), Value(0))
+
+
 def get_machine_stats():
     nodes = Node.objects.all()
     machines = nodes.filter(node_type=NODE_TYPE.MACHINE)
     # Rather overall amount of stats for machines.
     return machines.aggregate(
-        total_cpu=Sum('cpu_count'), total_mem=Sum('memory'),
-        total_storage=Sum('blockdevice__size'))
+        total_cpu=NotNullSum('cpu_count'),
+        total_mem=NotNullSum('memory'),
+        total_storage=NotNullSum('blockdevice__size'))
 
 
 def get_machine_state_stats():
@@ -85,12 +92,7 @@ def get_machines_by_architecture():
             dict(
                 short_arch="SUBSTRING(architecture FROM '(.*)/')")
             ).values_list('short_arch', flat=True)
-
-    count_by_arch = defaultdict(int)
-    for arch in node_arches:
-        count_by_arch[arch] += 1
-
-    return count_by_arch
+    return Counter(node_arches)
 
 
 def get_kvm_pods_stats():
@@ -99,8 +101,9 @@ def get_kvm_pods_stats():
     # total_mem is in MB
     # local_storage is in bytes
     available_resources = pods.aggregate(
-        cores=Sum('cores'), memory=Sum('memory'),
-        storage=Sum('local_storage'))
+        cores=NotNullSum('cores'),
+        memory=NotNullSum('memory'),
+        storage=NotNullSum('local_storage'))
 
     # available resources with overcommit
     over_cores = over_memory = 0
diff --git a/src/maasserver/tests/test_stats.py b/src/maasserver/tests/test_stats.py
index 65d9ea9..de36c50 100644
--- a/src/maasserver/tests/test_stats.py
+++ b/src/maasserver/tests/test_stats.py
@@ -113,6 +113,24 @@ class TestMAASStats(MAASServerTestCase):
         }
         self.assertEquals(compare, stats)
 
+    def test_get_kvm_pods_stats_no_pod(self):
+        self.assertEqual(
+            get_kvm_pods_stats(),
+            {'kvm_pods': 0,
+             'kvm_machines': 0,
+             'kvm_available_resources': {
+                 'cores': 0,
+                 'memory': 0,
+                 'storage': 0,
+                 'over_cores': 0,
+                 'over_memory': 0
+             },
+             'kvm_utilized_resources': {
+                 'cores': 0,
+                 'memory': 0,
+                 'storage': 0
+             }})
+
     def test_get_maas_stats(self):
         # Make one component of everything
         factory.make_RegionRackController()
@@ -181,6 +199,45 @@ class TestMAASStats(MAASServerTestCase):
         }
         self.assertEquals(stats, json.dumps(compare))
 
+    def test_get_maas_stats_no_machines(self):
+        expected = {
+            "controllers": {
+                "regionracks": 0,
+                "regions": 0,
+                "racks": 0,
+            },
+            "nodes": {
+                "machines": 0,
+                "devices": 0,
+            },
+            "machine_stats": {
+                "total_cpu": 0,
+                "total_mem": 0,
+                "total_storage": 0,
+            },
+            "machine_status": {
+                "new": 0,
+                "ready": 0,
+                "allocated": 0,
+                "deployed": 0,
+                "commissioning": 0,
+                "testing": 0,
+                "deploying": 0,
+                "failed_deployment": 0,
+                "failed_commissioning": 0,
+                "failed_testing": 0,
+                "broken": 0,
+            },
+            "network_stats": {
+                "spaces": 0,
+                "fabrics": 1,
+                "vlans": 1,
+                "subnets_v4": 0,
+                "subnets_v6": 0,
+            },
+        }
+        self.assertEqual(json.loads(get_maas_stats()), expected)
+
     def test_get_request_params_returns_params(self):
         factory.make_RegionRackController()
         params = {