Merge ~ltrager/maas:lp1878923_lp1878685 into maas:master

[Lee Trager]

LP: #1878923, #1878685 - Fix incorrect memory detection.

LXD was only detecting the amount of available memory in user space.
Memory reserved by the kernel was not shown as available. This caused MAAS
to show slightly less RAM then the system actually had.

MAAS was assuming that a NUMA node always had memory assigned to it. When
0 memory was assigned to a node commissioning would raise an exception and
fail.

[MAAS Lander (maas-lander)]

UNIT TESTS
-b lp1878923_lp1878685 lp:~ltrager/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: SUCCESS
COMMIT: 5c22723945be2ffa8359f9d50569030cf2fb93fd

review: Approve

[MAAS Lander (maas-lander)]

UNIT TESTS
-b lp1878923_lp1878685 lp:~ltrager/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: SUCCESS
COMMIT: e92ccb39d681fdb00906faf882c5af0b9fbb2057

review: Approve

[Dougal Matthews (d0ugal)]

lgtm

review: Approve

diff --git a/src/machine-resources/src/machine-resources/Gopkg.lock b/src/machine-resources/src/machine-resources/Gopkg.lock
index e4b0113..6c2f652 100644
--- a/src/machine-resources/src/machine-resources/Gopkg.lock
+++ b/src/machine-resources/src/machine-resources/Gopkg.lock
@@ -24,9 +24,10 @@
   revision = "3fe23663418fc1d724868c84f21b7519bbac7441"
 
 [[projects]]
+  branch = "master"
   name = "github.com/lxc/lxd"
   packages = ["lxd/resources","shared","shared/api","shared/cancel","shared/ioprogress","shared/logger","shared/osarch","shared/units","shared/usbid","shared/version"]
-  version = "lxd-4.0.1"
+  revision = "7516f0a2968b025efd1ea5066e78b16d6646b1c8"
 
 [[projects]]
   name = "github.com/mitchellh/go-homedir"
diff --git a/src/metadataserver/builtin_scripts/hooks.py b/src/metadataserver/builtin_scripts/hooks.py
index a65d06a..76c2614 100644
--- a/src/metadataserver/builtin_scripts/hooks.py
+++ b/src/metadataserver/builtin_scripts/hooks.py
@@ -438,7 +438,7 @@ def _process_lxd_resources(node, data):
         data
     )
     # Update memory.
-    node.memory, numa_nodes = _parse_memory(data, numa_nodes)
+    node.memory, numa_nodes = _parse_memory(data.get("memory", {}), numa_nodes)
 
     # Create or update NUMA nodes.
     numa_nodes = [
@@ -470,17 +470,16 @@ def _process_lxd_resources(node, data):
     _process_system_information(node, data.get("system", {}))
 
 
-def _parse_memory(data, numa_nodes):
-    total_memory = data.get("memory", {}).get("total", 0)
+def _parse_memory(memory, numa_nodes):
+    total_memory = memory.get("total", 0)
     default_numa_node = {"numa_node": 0, "total": total_memory}
-    for memory_node in data.get("memory", {}).get(
-        "nodes", [default_numa_node]
-    ):
-        numa_nodes[memory_node["numa_node"]]["memory"] = (
-            memory_node["total"] / 1024 / 1024
+
+    for memory_node in memory.get("nodes", [default_numa_node]):
+        numa_nodes[memory_node["numa_node"]]["memory"] = int(
+            memory_node.get("total", 0) / 1024 ** 2
         )
 
-    return total_memory / 1024 / 1024, numa_nodes
+    return int(total_memory / 1024 ** 2), numa_nodes
 
 
 def get_tags_from_block_info(block_info):
diff --git a/src/metadataserver/builtin_scripts/tests/test_hooks.py b/src/metadataserver/builtin_scripts/tests/test_hooks.py
index dfc7d89..de3424a 100644
--- a/src/metadataserver/builtin_scripts/tests/test_hooks.py
+++ b/src/metadataserver/builtin_scripts/tests/test_hooks.py
@@ -1412,6 +1412,42 @@ class TestProcessLXDResults(MAASServerTestCase):
         for numa_node in numa_nodes:
             self.assertEqual(int(total_memory / 1024 / 1024), numa_node.memory)
 
+    def test__accepts_numa_node_zero_memory(self):
+        # Regression test for LP:1878923
+        node = factory.make_Node()
+        self.patch(hooks_module, "update_node_network_information")
+        data = make_lxd_output()
+        data["resources"]["memory"] = {
+            "nodes": [
+                {
+                    "numa_node": 0,
+                    "hugepages_used": 0,
+                    "hugepages_total": 0,
+                    "used": 1314131968,
+                    "total": 33720463360,
+                },
+                {
+                    "numa_node": 1,
+                    "hugepages_used": 0,
+                    "hugepages_total": 0,
+                    "used": 0,
+                    "total": 0,
+                },
+            ],
+            "hugepages_total": 0,
+            "hugepages_used": 0,
+            "hugepages_size": 2097152,
+            "used": 602902528,
+            "total": 33720463360,
+        }
+
+        process_lxd_results(node, json.dumps(data).encode(), 0)
+
+        self.assertEquals(32158, node.memory)
+        self.assertItemsEqual(
+            [32158, 0], [numa.memory for numa in node.numanode_set.all()]
+        )
+
     def test__updates_cpu_numa_nodes(self):
         node = factory.make_Node()
         self.patch(hooks_module, "update_node_network_information")