Merge ~bjornt/maas:bug-1873430-2.7 into maas:2.7

[Björn Tillenius]

LP #1873430: Subnet pages take a long time to load.

Speed up space.list websocket handler when there are many ip addresses.

The space.list handler prefetched staticipaddress, which make scales
very badly with the number of statipaddress records.

Added tests to ensure that the number of queries is constant and
modified the prefetch and logic to make it pass.

Also remove unneeded prefetch for fabric.list websocket call.

(cherry picked from commit 4cffcfabb2c9e4bfedd8d05c67cdffc2da01ff2a)

[Björn Tillenius (bjornt)]

Self-approve backport.

review: Approve

diff --git a/src/maasserver/websockets/handlers/fabric.py b/src/maasserver/websockets/handlers/fabric.py
index 2c70ec3..65d5529 100644
--- a/src/maasserver/websockets/handlers/fabric.py
+++ b/src/maasserver/websockets/handlers/fabric.py
@@ -15,9 +15,7 @@ from maasserver.websockets.handlers.timestampedmodel import (
 
 class FabricHandler(TimestampedModelHandler):
     class Meta:
-        queryset = Fabric.objects.all().prefetch_related(
-            "vlan_set__interface_set"
-        )
+        queryset = Fabric.objects.all().prefetch_related("vlan_set")
         pk = "id"
         form = FabricForm
         form_requires_request = False
diff --git a/src/maasserver/websockets/handlers/space.py b/src/maasserver/websockets/handlers/space.py
index ac4c715..f31656c 100644
--- a/src/maasserver/websockets/handlers/space.py
+++ b/src/maasserver/websockets/handlers/space.py
@@ -5,6 +5,8 @@
 
 __all__ = ["SpaceHandler"]
 
+import itertools
+
 from maasserver.forms.space import SpaceForm
 from maasserver.models.space import Space
 from maasserver.permissions import NodePermission
@@ -15,9 +17,7 @@ from maasserver.websockets.handlers.timestampedmodel import (
 
 class SpaceHandler(TimestampedModelHandler):
     class Meta:
-        queryset = Space.objects.all().prefetch_related(
-            "vlan_set__subnet_set__staticipaddress_set__interface_set"
-        )
+        queryset = Space.objects.all().prefetch_related("vlan_set__subnet_set")
         pk = "id"
         form = SpaceForm
         form_requires_request = False
@@ -33,11 +33,15 @@ class SpaceHandler(TimestampedModelHandler):
 
     def dehydrate(self, obj, data, for_list=False):
         data["name"] = obj.get_name()
-        data["vlan_ids"] = list(
-            obj.vlan_set.order_by("id").values_list("id", flat=True)
-        )
-        data["subnet_ids"] = list(
-            obj.subnet_set.order_by("id").values_list("id", flat=True)
+        vlans = obj.vlan_set.all()
+        data["vlan_ids"] = sorted(vlan.id for vlan in vlans)
+        data["subnet_ids"] = sorted(
+            itertools.chain(
+                *[
+                    [subnet.id for subnet in vlan.subnet_set.all()]
+                    for vlan in vlans
+                ]
+            )
         )
         return data
 
diff --git a/src/maasserver/websockets/handlers/tests/test_fabric.py b/src/maasserver/websockets/handlers/tests/test_fabric.py
index b7e35dc..084469b 100644
--- a/src/maasserver/websockets/handlers/tests/test_fabric.py
+++ b/src/maasserver/websockets/handlers/tests/test_fabric.py
@@ -10,6 +10,7 @@ from maasserver.testing.factory import factory
 from maasserver.testing.testcase import MAASServerTestCase
 from maasserver.websockets.base import dehydrate_datetime
 from maasserver.websockets.handlers.fabric import FabricHandler
+from maastesting.djangotestcase import count_queries
 
 
 class TestFabricHandler(MAASServerTestCase):
@@ -61,3 +62,20 @@ class TestFabricHandler(MAASServerTestCase):
             self.dehydrate_fabric(fabric) for fabric in Fabric.objects.all()
         ]
         self.assertItemsEqual(expected_fabrics, handler.list({}))
+
+    def test_list_constant_queries(self):
+        user = factory.make_User()
+        handler = FabricHandler(user, {}, None)
+        for _ in range(10):
+            fabric = factory.make_Fabric()
+            vlan = fabric.get_default_vlan()
+            for _ in range(3):
+                node = factory.make_Node(interface=True)
+                interface = node.get_boot_interface()
+                interface.vlan = vlan
+                interface.save()
+
+        queries_one, _ = count_queries(handler.list, {"limit": 1})
+        queries_multiple, _ = count_queries(handler.list, {})
+
+        self.assertEqual(queries_one, queries_multiple)
diff --git a/src/maasserver/websockets/handlers/tests/test_space.py b/src/maasserver/websockets/handlers/tests/test_space.py
index e72d7d0..7f0df34 100644
--- a/src/maasserver/websockets/handlers/tests/test_space.py
+++ b/src/maasserver/websockets/handlers/tests/test_space.py
@@ -14,6 +14,7 @@ from maasserver.testing.testcase import MAASServerTestCase
 from maasserver.utils.orm import reload_object
 from maasserver.websockets.base import dehydrate_datetime
 from maasserver.websockets.handlers.space import SpaceHandler
+from maastesting.djangotestcase import count_queries
 
 
 class TestSpaceHandler(MAASServerTestCase):
@@ -53,6 +54,20 @@ class TestSpaceHandler(MAASServerTestCase):
         ]
         self.assertItemsEqual(expected_spaces, handler.list({}))
 
+    def test_list_constant_queries(self):
+        user = factory.make_User()
+        handler = SpaceHandler(user, {}, None)
+        for _ in range(10):
+            space = factory.make_Space()
+            node = factory.make_Node(interface=True)
+            interface = node.get_boot_interface()
+            subnet = factory.make_Subnet(space=space, vlan=interface.vlan)
+            factory.make_StaticIPAddress(subnet=subnet, interface=interface)
+        queries_one, _ = count_queries(handler.list, {"limit": 1})
+        queries_multiple, _ = count_queries(handler.list, {})
+
+        self.assertEqual(queries_one, queries_multiple)
+
 
 class TestSpaceHandlerDelete(MAASServerTestCase):
     def test__delete_as_admin_success(self):