Merge ~ltrager/maas:node_device_websocket into maas:master

[Lee Trager]

Add NodeDevice websocket

[MAAS Lander (maas-lander)]

UNIT TESTS
-b node_device_websocket lp:~ltrager/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED
LOG: http://maas-ci.internal:8080/job/maas/job/branch-tester/8908/console
COMMIT: 4cc0b29b718e96d3a7c82ec7b88e1174c53d9003

review: Needs Fixing

[Alberto Donato (ack)]

LGTM

review: Approve

525610a... by Lee Trager on 2020-12-16

Merge branch 'master' into node_device_model

b48316d... by Lee Trager on 2020-12-16

ack fix

2fbf073... by Lee Trager on 2020-12-16

Merge branch 'node_device_model' into process_node_device_data

4d8aa9f... by Lee Trager on 2020-12-16

Fix merge mistake

69c462f... by Lee Trager on 2020-12-16

Merge branch 'process_node_device_data' into node_device_api

f69a752... by Lee Trager on 2020-12-16

Merge branch 'node_device_api' into node_device_websocket

[MAAS Lander (maas-lander)]

UNIT TESTS
-b node_device_websocket lp:~ltrager/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED
LOG: http://maas-ci.internal:8080/job/maas/job/branch-tester/8923/console
COMMIT: f69a7522c82a1e51a1f29e3097902aff3cd37654

review: Needs Fixing

72cf97b... by Lee Trager on 2020-12-18

Merge branch 'master' into process_node_device_data

1d81711... by Lee Trager on 2020-12-18

Merge branch 'process_node_device_data' into node_device_api

174a1d0... by Lee Trager on 2020-12-18

Merge branch 'node_device_api' into node_device_websocket

2b01786... by Lee Trager on 2020-12-18

Fix failing test

[MAAS Lander (maas-lander)]

UNIT TESTS
-b node_device_websocket lp:~ltrager/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED
LOG: http://maas-ci.internal:8080/job/maas/job/branch-tester/8949/console
COMMIT: 2b01786670299787dc01bae90e4a0b14851ec43b

review: Needs Fixing

ad87d83... by Lee Trager on 2021-01-04

Merge branch 'master' into process_node_device_data

917084b... by Lee Trager on 2021-01-05

adam-collard fixes

f7b6424... by Lee Trager on 2021-01-05

Merge branch 'process_node_device_data' into node_device_api

d6d84b1... by Lee Trager on 2021-01-05

Merge branch 'node_device_api' into node_device_websocket

[MAAS Lander (maas-lander)]

UNIT TESTS
-b node_device_websocket lp:~ltrager/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: SUCCESS
COMMIT: d6d84b1ebcbb150d596d9d4dff11ddbfe0a18e89

review: Approve

diff --git a/src/maasserver/triggers/testing.py b/src/maasserver/triggers/testing.py
index 0a52dbd..5525281 100644
--- a/src/maasserver/triggers/testing.py
+++ b/src/maasserver/triggers/testing.py
@@ -1,4 +1,4 @@
-# Copyright 2016 Canonical Ltd.  This software is licensed under the
+# Copyright 2016-2020 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Helper class for all tests using the `PostgresListenerService` under
@@ -29,6 +29,7 @@ from maasserver.models.filesystemgroup import FilesystemGroup
 from maasserver.models.interface import Interface
 from maasserver.models.iprange import IPRange
 from maasserver.models.node import Node, RackController, RegionController
+from maasserver.models.nodedevice import NodeDevice
 from maasserver.models.nodemetadata import NodeMetadata
 from maasserver.models.packagerepository import PackageRepository
 from maasserver.models.partition import Partition
@@ -827,6 +828,21 @@ class TransactionalHelpersMixin:
         config.value = value
         config.save()
 
+    @transactional
+    def create_node_device(self, params=None):
+        if params is None:
+            params = {}
+        return factory.make_NodeDevice(**params)
+
+    @transactional
+    def update_node_device(self, id, params, **kwargs):
+        return apply_update_to_model(NodeDevice, id, params, **kwargs)
+
+    @transactional
+    def delete_node_device(self, id):
+        node_device = NodeDevice.objects.get(id=id)
+        node_device.delete()
+
 
 class DNSHelpersMixin:
     """Helper to get the zone serial and to assert it was incremented."""
diff --git a/src/maasserver/triggers/tests/test_init.py b/src/maasserver/triggers/tests/test_init.py
index 165d325..93e749d 100644
--- a/src/maasserver/triggers/tests/test_init.py
+++ b/src/maasserver/triggers/tests/test_init.py
@@ -187,6 +187,9 @@ class TestTriggersUsed(MAASServerTestCase):
         "neighbour_neighbour_create_notify",
         "neighbour_neighbour_delete_notify",
         "neighbour_neighbour_update_notify",
+        "nodedevice_nodedevice_create_notify",
+        "nodedevice_nodedevice_delete_notify",
+        "nodedevice_nodedevice_update_notify",
         "node_device_create_notify",
         "node_device_delete_notify",
         "node_device_update_notify",
diff --git a/src/maasserver/triggers/tests/test_websocket_listener.py b/src/maasserver/triggers/tests/test_websocket_listener.py
index 8589e21..9b1f535 100644
--- a/src/maasserver/triggers/tests/test_websocket_listener.py
+++ b/src/maasserver/triggers/tests/test_websocket_listener.py
@@ -5132,3 +5132,66 @@ class TestScriptListener(
             self.assertEqual(("delete", "%s" % script.id), dv.value)
         finally:
             yield listener.stopService()
+
+
+class TestNodeDeviceListener(
+    MAASTransactionServerTestCase, TransactionalHelpersMixin
+):
+    @wait_for_reactor
+    @inlineCallbacks
+    def test_calls_handler_on_create_notification(self):
+        yield deferToDatabase(register_websocket_triggers)
+        node = yield deferToDatabase(self.create_node)
+        listener = self.make_listener_without_delay()
+        dv = DeferredValue()
+        listener.register("nodedevice", lambda *args: dv.set(args))
+        yield listener.startService()
+        try:
+            node_device = yield deferToDatabase(
+                self.create_node_device, {"node": node}
+            )
+            yield dv.get(timeout=2)
+            self.assertEqual(("create", "%s" % node_device.id), dv.value)
+        finally:
+            yield listener.stopService()
+
+    @wait_for_reactor
+    @inlineCallbacks
+    def test_calls_handler_on_update_notification(self):
+        yield deferToDatabase(register_websocket_triggers)
+        listener = self.make_listener_without_delay()
+        dv = DeferredValue()
+        listener.register("nodedevice", lambda *args: dv.set(args))
+        node_device = yield deferToDatabase(self.create_node_device)
+
+        yield listener.startService()
+        try:
+            yield deferToDatabase(
+                self.update_node_device,
+                node_device.id,
+                {
+                    "commissioning_driver": factory.make_name(
+                        "commissioning_driver"
+                    )
+                },
+            )
+            yield dv.get(timeout=2)
+            self.assertEqual(("update", "%s" % node_device.id), dv.value)
+        finally:
+            yield listener.stopService()
+
+    @wait_for_reactor
+    @inlineCallbacks
+    def test_calls_handler_on_delete_notification(self):
+        yield deferToDatabase(register_websocket_triggers)
+        listener = self.make_listener_without_delay()
+        dv = DeferredValue()
+        listener.register("nodedevice", lambda *args: dv.set(args))
+        node_device = yield deferToDatabase(self.create_node_device)
+        yield listener.startService()
+        try:
+            yield deferToDatabase(self.delete_node_device, node_device.id)
+            yield dv.get(timeout=2)
+            self.assertEqual(("delete", "%s" % node_device.id), dv.value)
+        finally:
+            yield listener.stopService()
diff --git a/src/maasserver/triggers/websocket.py b/src/maasserver/triggers/websocket.py
index bbd0861..ac2f9e2 100644
--- a/src/maasserver/triggers/websocket.py
+++ b/src/maasserver/triggers/websocket.py
@@ -1,4 +1,4 @@
-# Copyright 2015-2018 Canonical Ltd.  This software is licensed under the
+# Copyright 2015-2020 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """
@@ -2724,3 +2724,21 @@ def register_websocket_triggers():
         )
     )
     register_triggers("metadataserver_script", "script")
+
+    # NodeDevice table
+    register_procedure(
+        render_notification_procedure(
+            "nodedevice_create_notify", "nodedevice_create", "NEW.id"
+        )
+    )
+    register_procedure(
+        render_notification_procedure(
+            "nodedevice_update_notify", "nodedevice_update", "NEW.id"
+        )
+    )
+    register_procedure(
+        render_notification_procedure(
+            "nodedevice_delete_notify", "nodedevice_delete", "OLD.id"
+        )
+    )
+    register_triggers("maasserver_nodedevice", "nodedevice")
diff --git a/src/maasserver/websockets/handlers/__init__.py b/src/maasserver/websockets/handlers/__init__.py
index 87901f8..1ae365c 100644
--- a/src/maasserver/websockets/handlers/__init__.py
+++ b/src/maasserver/websockets/handlers/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2015-2017 Canonical Ltd.  This software is licensed under the
+# Copyright 2015-2020 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Handlers for the WebSocket connections."""
@@ -19,7 +19,7 @@ __all__ = [
     "IPRangeHandler",
     "IPRangeHandler",
     "MachineHandler",
-    "NodeResultHandler",
+    "NodeDeviceHandler",
     "NodeResultHandler",
     "NotificationHandler",
     "PackageRepositoryHandler",
@@ -52,6 +52,7 @@ from maasserver.websockets.handlers.fabric import FabricHandler
 from maasserver.websockets.handlers.general import GeneralHandler
 from maasserver.websockets.handlers.iprange import IPRangeHandler
 from maasserver.websockets.handlers.machine import MachineHandler
+from maasserver.websockets.handlers.node_device import NodeDeviceHandler
 from maasserver.websockets.handlers.node_result import NodeResultHandler
 from maasserver.websockets.handlers.notification import NotificationHandler
 from maasserver.websockets.handlers.packagerepository import (
diff --git a/src/maasserver/websockets/handlers/node.py b/src/maasserver/websockets/handlers/node.py
index 48b4e38..bca7d2f 100644
--- a/src/maasserver/websockets/handlers/node.py
+++ b/src/maasserver/websockets/handlers/node.py
@@ -138,7 +138,7 @@ class NodeHandler(TimestampedModelHandler):
     def dehydrate_numanode(self, numa_node):
         details = {
             attr: getattr(numa_node, attr)
-            for attr in ("index", "memory", "cores")
+            for attr in ("id", "index", "memory", "cores")
         }
         details["hugepages_set"] = [
             self.dehydrate_hugepages(hugepages)
diff --git a/src/maasserver/websockets/handlers/node_device.py b/src/maasserver/websockets/handlers/node_device.py
new file mode 100644
index 0000000..c69409d
--- /dev/null
+++ b/src/maasserver/websockets/handlers/node_device.py
@@ -0,0 +1,51 @@
+# Copyright 2020 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""The Script handler for the WebSocket connection."""
+
+from operator import attrgetter
+
+from maasserver.models import NodeDevice
+from maasserver.websockets.base import HandlerPKError
+from maasserver.websockets.handlers.timestampedmodel import (
+    TimestampedModelHandler,
+)
+
+
+class NodeDeviceHandler(TimestampedModelHandler):
+    class Meta:
+        queryset = NodeDevice.objects.all()
+        pk = "id"
+        allowed_methods = ["list", "delete"]
+        listen_channels = ["nodedevice"]
+
+    def dehydrate(self, obj, data, for_list=False):
+        # When NodeDevices are loaded in the UI the client has already received
+        # the keys below. Instead of reprocessing them make it clear the handler
+        # is only returning the ids, not values,
+        for key in [
+            "physical_blockdevice",
+            "physical_interface",
+            "numa_node",
+            "node",
+        ]:
+            id = data.pop(key)
+            data[f"{key}_id"] = id
+        return data
+
+    def list(self, params):
+        """List NodeDevice objects.
+
+        :param system_id: `Node.system_id` for the NodeDevices.
+        """
+        if "system_id" not in params:
+            raise HandlerPKError("Missing system_id in params")
+        system_id = params["system_id"]
+
+        qs = self.get_queryset(for_list=True)
+        qs = qs.filter(node__system_id=system_id)
+
+        objs = list(qs)
+        getpk = attrgetter(self._meta.pk)
+        self.cache["loaded_pks"].update(getpk(obj) for obj in objs)
+        return [self.full_dehydrate(obj, for_list=True) for obj in objs]
diff --git a/src/maasserver/websockets/handlers/tests/test_machine.py b/src/maasserver/websockets/handlers/tests/test_machine.py
index 6ce5614..62cd7d6 100644
--- a/src/maasserver/websockets/handlers/tests/test_machine.py
+++ b/src/maasserver/websockets/handlers/tests/test_machine.py
@@ -2088,20 +2088,29 @@ class TestMachineHandler(MAASServerTestCase):
         self.assertEqual(
             result["numa_nodes"],
             [
-                {"index": 0, "memory": 0, "cores": [], "hugepages_set": []},
                 {
+                    "id": numa_node.id - 3,
+                    "index": 0,
+                    "memory": 0,
+                    "cores": [],
+                    "hugepages_set": [],
+                },
+                {
+                    "id": numa_node.id - 2,
                     "index": 1,
                     "memory": 512,
                     "cores": [0, 1],
                     "hugepages_set": [{"page_size": 1024, "total": 1024}],
                 },
                 {
+                    "id": numa_node.id - 1,
                     "index": 2,
                     "memory": 1024,
                     "cores": [2, 3],
                     "hugepages_set": [{"page_size": 1024, "total": 2048}],
                 },
                 {
+                    "id": numa_node.id,
                     "index": 3,
                     "memory": 2048,
                     "cores": [4, 5],
diff --git a/src/maasserver/websockets/handlers/tests/test_node_device.py b/src/maasserver/websockets/handlers/tests/test_node_device.py
new file mode 100644
index 0000000..beea5dd
--- /dev/null
+++ b/src/maasserver/websockets/handlers/tests/test_node_device.py
@@ -0,0 +1,51 @@
+# Copyright 2020 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Tests for `maasserver.websockets.handlers.node_device`"""
+
+from maasserver.testing.factory import factory
+from maasserver.testing.testcase import MAASServerTestCase
+from maasserver.websockets.base import dehydrate_datetime, HandlerPKError
+from maasserver.websockets.handlers.node_device import NodeDeviceHandler
+
+
+class TestNodeDeviceHandler(MAASServerTestCase):
+    def dehydrate_node_device(self, node_device):
+        return {
+            "id": node_device.id,
+            "created": dehydrate_datetime(node_device.created),
+            "updated": dehydrate_datetime(node_device.updated),
+            "bus": node_device.bus,
+            "hardware_type": node_device.hardware_type,
+            "vendor_id": node_device.vendor_id,
+            "product_id": node_device.product_id,
+            "vendor_name": node_device.vendor_name,
+            "product_name": node_device.product_name,
+            "commissioning_driver": node_device.commissioning_driver,
+            "bus_number": node_device.bus_number,
+            "device_number": node_device.device_number,
+            "pci_address": node_device.pci_address,
+            "physical_blockdevice_id": node_device.physical_blockdevice_id,
+            "physical_interface_id": node_device.physical_interface_id,
+            "numa_node_id": node_device.numa_node_id,
+            "node_id": node_device.node_id,
+        }
+
+    def test_list(self):
+        user = factory.make_User()
+        handler = NodeDeviceHandler(user, {}, None)
+        node = factory.make_Node_with_Interface_on_Subnet()
+        factory.make_Node()
+
+        self.assertEqual(
+            [
+                self.dehydrate_node_device(node_device)
+                for node_device in node.node_devices.all()
+            ],
+            handler.list({"system_id": node.system_id}),
+        )
+
+    def test_list_raises_error_if_no_system_id(self):
+        user = factory.make_User()
+        handler = NodeDeviceHandler(user, {}, None)
+        self.assertRaises(HandlerPKError, handler.list, {})
diff --git a/src/maasserver/websockets/tests/test_protocol.py b/src/maasserver/websockets/tests/test_protocol.py
index 3007e43..b2cba6a 100644
--- a/src/maasserver/websockets/tests/test_protocol.py
+++ b/src/maasserver/websockets/tests/test_protocol.py
@@ -775,6 +775,7 @@ ALL_NOTIFIERS = (
     "fabric",
     "iprange",
     "machine",
+    "nodedevice",
     "notification",
     "notificationdismissal",
     "packagerepository",
@@ -809,6 +810,7 @@ ALL_HANDLERS = (
     "general",
     "iprange",
     "machine",
+    "nodedevice",
     "noderesult",
     "notification",
     "packagerepository",