Merge ~ltrager/maas:lp1878923_lp1878685_2.7 into maas:2.7

[Lee Trager]

Backport of 894a324 - LP: #1878923, #1878685 - Fix incorrect memory detection.

LXD was only detecting the amount of available memory in user space.
Memory reserved by the kernel was not shown as available. This caused MAAS
to show slightly less RAM then the system actually had.

MAAS was assuming that a NUMA node always had memory assigned to it. When
0 memory was assigned to a node commissioning would raise an exception and
fail.

[Lee Trager (ltrager)]

Approved in https://code.launchpad.net/~ltrager/maas/+git/maas/+merge/384202

review: Approve

diff --git a/src/machine-resources/src/machine-resources/Gopkg.lock b/src/machine-resources/src/machine-resources/Gopkg.lock
index a96f98c..3006552 100644
--- a/src/machine-resources/src/machine-resources/Gopkg.lock
+++ b/src/machine-resources/src/machine-resources/Gopkg.lock
@@ -20,10 +20,15 @@
   version = "0.5.0"
 
 [[projects]]
+  name = "github.com/juju/errors"
+  packages = ["."]
+  revision = "3fe23663418fc1d724868c84f21b7519bbac7441"
+
+[[projects]]
   branch = "master"
   name = "github.com/lxc/lxd"
-  packages = ["lxd/resources","shared","shared/api","shared/cancel","shared/ioprogress","shared/logger","shared/units","shared/usbid"]
-  revision = "7b12be0dc6b099dffc369299af2803a77611fc8d"
+  packages = ["lxd/resources","shared","shared/api","shared/cancel","shared/ioprogress","shared/logger","shared/osarch","shared/units","shared/usbid","shared/version"]
+  revision = "7516f0a2968b025efd1ea5066e78b16d6646b1c8"
 
 [[projects]]
   name = "github.com/mitchellh/go-homedir"
diff --git a/src/metadataserver/builtin_scripts/hooks.py b/src/metadataserver/builtin_scripts/hooks.py
index d301090..c0d3b2a 100644
--- a/src/metadataserver/builtin_scripts/hooks.py
+++ b/src/metadataserver/builtin_scripts/hooks.py
@@ -450,7 +450,7 @@ def process_lxd_results(node, output, exit_status):
         data
     )
     # Update memory.
-    node.memory, numa_nodes = _parse_memory(data, numa_nodes)
+    node.memory, numa_nodes = _parse_memory(data.get("memory", {}), numa_nodes)
 
     # Create or update NUMA nodes.
     numa_nodes = [
@@ -543,18 +543,16 @@ def _parse_cpuinfo(data):
     return cpu_count, cpu_speed, cpu_model, numa_nodes
 
 
-def _parse_memory(data, numa_nodes):
-
-    total_memory = data.get("memory", {}).get("total", 0)
+def _parse_memory(memory, numa_nodes):
+    total_memory = memory.get("total", 0)
     default_numa_node = {"numa_node": 0, "total": total_memory}
-    for memory_node in data.get("memory", {}).get(
-        "nodes", [default_numa_node]
-    ):
-        numa_nodes[memory_node["numa_node"]]["memory"] = (
-            memory_node["total"] / 1024 / 1024
+
+    for memory_node in memory.get("nodes", [default_numa_node]):
+        numa_nodes[memory_node["numa_node"]]["memory"] = int(
+            memory_node.get("total", 0) / 1024 ** 2
         )
 
-    return total_memory / 1024 / 1024, numa_nodes
+    return int(total_memory / 1024 ** 2), numa_nodes
 
 
 def set_virtual_tag(node, output, exit_status):
diff --git a/src/metadataserver/builtin_scripts/tests/test_hooks.py b/src/metadataserver/builtin_scripts/tests/test_hooks.py
index e5f6fa9..423e0e4 100644
--- a/src/metadataserver/builtin_scripts/tests/test_hooks.py
+++ b/src/metadataserver/builtin_scripts/tests/test_hooks.py
@@ -1659,6 +1659,42 @@ class TestProcessLXDResults(MAASServerTestCase):
         for numa_node in numa_nodes:
             self.assertEqual(int(total_memory / 1024 / 1024), numa_node.memory)
 
+    def test__accepts_numa_node_zero_memory(self):
+        # Regression test for LP:1878923
+        node = factory.make_Node()
+        self.patch(hooks_module, "update_node_network_information")
+        data = deepcopy(SAMPLE_LXD_JSON)
+        data["memory"] = {
+            "nodes": [
+                {
+                    "numa_node": 0,
+                    "hugepages_used": 0,
+                    "hugepages_total": 0,
+                    "used": 1314131968,
+                    "total": 33720463360,
+                },
+                {
+                    "numa_node": 1,
+                    "hugepages_used": 0,
+                    "hugepages_total": 0,
+                    "used": 0,
+                    "total": 0,
+                },
+            ],
+            "hugepages_total": 0,
+            "hugepages_used": 0,
+            "hugepages_size": 2097152,
+            "used": 602902528,
+            "total": 33720463360,
+        }
+
+        process_lxd_results(node, json.dumps(data).encode(), 0)
+
+        self.assertEquals(32158, node.memory)
+        self.assertItemsEqual(
+            [32158, 0], [numa.memory for numa in node.numanode_set.all()]
+        )
+
     def test__updates_cpu_numa_nodes(self):
         node = factory.make_Node()
         self.patch(hooks_module, "update_node_network_information")