Merge ~adam-collard/maas:lxd-numa-memory-1878923 into maas:master

[Adam Collard]

Improve robustness of NUMA node identification

Fixes LP:1885157

[MAAS Lander (maas-lander)]

UNIT TESTS
-b lxd-numa-memory-1878923 lp:~adam-collard/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED
LOG: http://maas-ci.internal:8080/job/maas/job/branch-tester/7820/console
COMMIT: 245d125f97afbe54d50fe792bf0412cb1f4f0214

review: Needs Fixing

[Lee Trager (ltrager)]

LGTM!

review: Approve

[MAAS Lander (maas-lander)]

UNIT TESTS
-b lxd-numa-memory-1878923 lp:~adam-collard/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: SUCCESS
COMMIT: 46dea51cc575a72e4d210606c04a50e6e845e42f

review: Approve

diff --git a/src/maasserver/models/numa.py b/src/maasserver/models/numa.py
index ee82464..f6e347c 100644
--- a/src/maasserver/models/numa.py
+++ b/src/maasserver/models/numa.py
@@ -19,6 +19,9 @@ class NUMANode(CleanSave, TimestampedModel):
     class Meta:
         unique_together = [("node", "index")]
 
+    def __repr__(self):
+        return f"<NUMANode of {self.index} {self.node!r} cores: {self.cores!r} {self.memory}>"
+
 
 def create_default_numanode(machine):
     """Create the default "0" NUMA node for a machine."""
diff --git a/src/metadataserver/builtin_scripts/hooks.py b/src/metadataserver/builtin_scripts/hooks.py
index c7a3580..26c7e11 100644
--- a/src/metadataserver/builtin_scripts/hooks.py
+++ b/src/metadataserver/builtin_scripts/hooks.py
@@ -464,10 +464,7 @@ def _process_lxd_resources(node, data):
         NUMANode.objects.update_or_create(
             node=node,
             index=numa_index,
-            defaults={
-                "memory": numa_data["memory"],
-                "cores": numa_data["cores"],
-            },
+            defaults={"memory": numa_data.memory, "cores": numa_data.cores},
         )[0]
         for numa_index, numa_data in numa_nodes.items()
     ]
@@ -494,7 +491,7 @@ def _parse_memory(memory, numa_nodes):
     default_numa_node = {"numa_node": 0, "total": total_memory}
 
     for memory_node in memory.get("nodes", [default_numa_node]):
-        numa_nodes[memory_node["numa_node"]]["memory"] = int(
+        numa_nodes[memory_node["numa_node"]].memory = int(
             memory_node.get("total", 0) / 1024 ** 2
         )
 
diff --git a/src/metadataserver/builtin_scripts/tests/test_hooks.py b/src/metadataserver/builtin_scripts/tests/test_hooks.py
index 872b5ae..be1a517 100644
--- a/src/metadataserver/builtin_scripts/tests/test_hooks.py
+++ b/src/metadataserver/builtin_scripts/tests/test_hooks.py
@@ -1486,6 +1486,53 @@ class TestProcessLXDResults(MAASServerTestCase):
             [32158, 0], [numa.memory for numa in node.numanode_set.all()]
         )
 
+    def test_updates_memory_no_corresponding_cpu_numa_node(self):
+        # Regression test for LP:1885157
+        node = factory.make_Node()
+        self.patch(hooks_module, "update_node_network_information")
+        data = make_lxd_output()
+        data["resources"]["memory"] = {
+            "nodes": [
+                {
+                    "numa_node": 252,
+                    "hugepages_used": 0,
+                    "hugepages_total": 0,
+                    "used": 1314131968,
+                    "total": 33720463360,
+                },
+                {
+                    "numa_node": 253,
+                    "hugepages_used": 0,
+                    "hugepages_total": 0,
+                    "used": 0,
+                    "total": 0,
+                },
+                {
+                    "numa_node": 254,
+                    "hugepages_used": 0,
+                    "hugepages_total": 0,
+                    "used": 0,
+                    "total": 0,
+                },
+                {
+                    "numa_node": 255,
+                    "hugepages_used": 0,
+                    "hugepages_total": 0,
+                    "used": 0,
+                    "total": 0,
+                },
+            ],
+            "hugepages_total": 0,
+            "hugepages_used": 0,
+            "hugepages_size": 2097152,
+            "used": 602902528,
+            "total": 33720463360,
+        }
+
+        process_lxd_results(node, json.dumps(data).encode(), 0)
+        numa_nodes = NUMANode.objects.filter(node=node).order_by("index")
+        self.assertEqual(6, len(numa_nodes))
+
     def test_updates_cpu_numa_nodes(self):
         node = factory.make_Node()
         self.patch(hooks_module, "update_node_network_information")
diff --git a/src/provisioningserver/utils/lxd.py b/src/provisioningserver/utils/lxd.py
index 0b29b64..f45ffd1 100644
--- a/src/provisioningserver/utils/lxd.py
+++ b/src/provisioningserver/utils/lxd.py
@@ -5,8 +5,11 @@
 
 __all__ = ["parse_lxd_cpuinfo"]
 
+from collections import defaultdict
 import re
 
+import attr
+
 
 # This is needed on the rack controller in the LXDPodDriver to set
 # the cpu_speed for discovered machines as the lxd resources are
@@ -16,6 +19,13 @@ def lxd_cpu_speed(data):
     return cpu_speed
 
 
+@attr.s
+class NUMANode:
+
+    memory: int = attr.ib(default=0)
+    cores: list = attr.ib(factory=list)
+
+
 def parse_lxd_cpuinfo(data):
     cpu_speed = 0
     cpu_model = None
@@ -23,7 +33,7 @@ def parse_lxd_cpuinfo(data):
     # Only update the cpu_model if all the socket names match.
     sockets = data.get("cpu", {}).get("sockets", [])
     names = []
-    numa_nodes = {}
+    numa_nodes = defaultdict(NUMANode)
     for socket in sockets:
         name = socket.get("name")
         if name is not None:
@@ -32,10 +42,7 @@ def parse_lxd_cpuinfo(data):
             for thread in core.get("threads", []):
                 thread_id = thread["id"]
                 numa_node = thread["numa_node"]
-                if numa_node in numa_nodes:
-                    numa_nodes[numa_node]["cores"].append(thread_id)
-                else:
-                    numa_nodes[numa_node] = {"cores": [thread_id]}
+                numa_nodes[numa_node].cores.append(thread_id)
 
     if len(names) > 0 and all(name == names[0] for name in names):
         cpu = names[0]