Merge ~ltrager/maas:ibm_z_partition into maas:master

[Lee Trager]

Add boot support for IBM Z Partitions.

[MAAS Lander (maas-lander)]

UNIT TESTS
-b ibm_z_partition lp:~ltrager/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED
LOG: http://maas-ci.internal:8080/job/maas/job/branch-tester/9272/console
COMMIT: fb7449dbfc652b26ea481179db00db429aa52520

review: Needs Fixing

[Adam Collard (adam-collard)]

jenkins: !test

[MAAS Lander (maas-lander)]

UNIT TESTS
-b ibm_z_partition lp:~ltrager/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: SUCCESS
COMMIT: fb7449dbfc652b26ea481179db00db429aa52520

review: Approve

[MAAS Lander (maas-lander)]

UNIT TESTS
-b ibm_z_partition lp:~ltrager/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: SUCCESS
COMMIT: d442febf09fb6f157f5d7460bbccf6298a9d1aa3

review: Approve

[Björn Tillenius (bjornt)]

+1

review: Approve

diff --git a/src/maasserver/models/node.py b/src/maasserver/models/node.py
index f58429f..b6f5008 100644
--- a/src/maasserver/models/node.py
+++ b/src/maasserver/models/node.py
@@ -1,4 +1,4 @@
-# Copyright 2012-2020 Canonical Ltd.  This software is licensed under the
+# Copyright 2012-2021 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Node objects."""
@@ -235,7 +235,9 @@ maaslog = get_maas_logger("node")
 
 # Holds the known `bios_boot_methods`. If `bios_boot_method` is not in this
 # list then it will fallback to `DEFAULT_BIOS_BOOT_METHOD`.
-KNOWN_BIOS_BOOT_METHODS = frozenset(["pxe", "uefi", "powernv", "powerkvm"])
+KNOWN_BIOS_BOOT_METHODS = frozenset(
+    ["pxe", "uefi", "powernv", "powerkvm", "s390x_partition"]
+)
 
 # Default `bios_boot_method`. See `KNOWN_BIOS_BOOT_METHOD` above for usage.
 DEFAULT_BIOS_BOOT_METHOD = "pxe"
diff --git a/src/maasserver/models/tests/test_node.py b/src/maasserver/models/tests/test_node.py
index 384cb4a..83489a4 100644
--- a/src/maasserver/models/tests/test_node.py
+++ b/src/maasserver/models/tests/test_node.py
@@ -1,4 +1,4 @@
-# Copyright 2012-2020 Canonical Ltd.  This software is licensed under the
+# Copyright 2012-2021 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Test maasserver models."""
@@ -1369,6 +1369,10 @@ class TestNode(MAASServerTestCase):
         node = factory.make_Node(bios_boot_method="powerkvm")
         self.assertEqual("powerkvm", node.get_bios_boot_method())
 
+    def test_get_bios_boot_method_returns_s390x_partition(self):
+        node = factory.make_Node(bios_boot_method="s390x_partition")
+        self.assertEqual("s390x_partition", node.get_bios_boot_method())
+
     def test_get_bios_boot_method_ipmi_efi_fallback(self):
         ipmi_efi = factory.make_BMC(
             power_type="ipmi",
diff --git a/src/provisioningserver/boot/__init__.py b/src/provisioningserver/boot/__init__.py
index d4b13e2..2e20863 100644
--- a/src/provisioningserver/boot/__init__.py
+++ b/src/provisioningserver/boot/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2012-2018 Canonical Ltd.  This software is licensed under the
+# Copyright 2012-2021 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Boot Methods."""
@@ -439,6 +439,9 @@ from provisioningserver.boot.pxe import PXEBootMethod  # noqa:E402 isort:skip
 from provisioningserver.boot.s390x import (  # noqa:E402 isort:skip
     S390XBootMethod,
 )
+from provisioningserver.boot.s390x_partition import (  # noqa:E402 isort:skip
+    S390XPartitionBootMethod,
+)
 from provisioningserver.boot.uefi_amd64 import (  # noqa:E402 isort:skip
     UEFIAMD64BootMethod,
 )
@@ -460,6 +463,7 @@ builtin_boot_methods = [
     PowerNVBootMethod(),
     WindowsPXEBootMethod(),
     S390XBootMethod(),
+    S390XPartitionBootMethod(),
 ]
 for method in builtin_boot_methods:
     BootMethodRegistry.register_item(method.name, method)
diff --git a/src/provisioningserver/boot/s390x_partition.py b/src/provisioningserver/boot/s390x_partition.py
new file mode 100644
index 0000000..c45e348
--- /dev/null
+++ b/src/provisioningserver/boot/s390x_partition.py
@@ -0,0 +1,80 @@
+# Copyright 2021 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""S390X DPM Partition Boot Method"""
+
+import tempita
+
+from provisioningserver.boot import BootMethod, BytesReader, get_remote_mac
+from provisioningserver.boot.s390x import format_bootif
+from provisioningserver.kernel_opts import compose_kernel_command_line
+
+
+class S390XPartitionBootMethod(BootMethod):
+
+    name = "s390x_partition"
+    bios_boot_method = "s390x_partition"
+    template_subdir = "s390x_partition"
+    # S390X partitions has its bootloader built into the firmware. The
+    # "bootloader" provided must be the bootloader configuration file. The
+    # file format is similar to pxelinux.cfg but supports limited options
+    # and requires an '=' to be used between keys and values.
+    # https://www.ibm.com/support/pages/sites/default/files/inline-files/SB10-7176-01.pdf
+    bootloader_path = "s390x_partition/maas"
+    arch_octet = "00:20"
+    user_class = None
+
+    def match_path(self, backend, path):
+        """Checks path for the configuration file that needs to be
+        generated.
+
+        :param backend: requesting backend
+        :param path: requested path
+        :return: dict of match params from path, None if no match
+        """
+        if path == self.bootloader_path.encode():
+            return {
+                "arch": "s390x",
+                "mac": get_remote_mac(),
+            }
+        else:
+            return None
+
+    def get_reader(self, backend, kernel_params, mac=None, **extra):
+        """Render a configuration file as a unicode string.
+
+        :param backend: requesting backend
+        :param kernel_params: An instance of `KernelParameters`.
+        :param mac: Optional MAC address discovered by `match_path`.
+        :param extra: Allow for other arguments. This is a safety valve;
+            parameters generated in another component (for example, see
+            `TFTPBackend.get_boot_method_reader`) won't cause this to break.
+        """
+        template = self.get_template(
+            kernel_params.purpose, kernel_params.arch, kernel_params.subarch
+        )
+        namespace = self.compose_template_namespace(kernel_params)
+
+        def kernel_command(params):
+            cmd_line = compose_kernel_command_line(params)
+            # Modify the kernel_command to inject the BOOTIF. S390X doesn't
+            # support the IPAPPEND pxelinux flag.
+            if mac is not None:
+                return "%s BOOTIF=%s" % (cmd_line, format_bootif(mac))
+            return cmd_line
+
+        namespace["kernel_command"] = kernel_command
+
+        # We are going to do 2 passes of tempita substitution because there
+        # may be things like kernel params which include variables that can
+        # only be populated at run time and thus contain variables themselves.
+        # For example, an OS may need a kernel parameter that points back to
+        # fs_host and the kernel parameter comes through as part of the simple
+        # stream.
+        step1 = template.substitute(namespace)
+        return BytesReader(
+            tempita.Template(step1).substitute(namespace).encode("utf-8")
+        )
+
+    def link_bootloader(self, destination):
+        """Does nothing. No extra boot files are required."""
diff --git a/src/provisioningserver/boot/tests/test_s390x_partition.py b/src/provisioningserver/boot/tests/test_s390x_partition.py
new file mode 100644
index 0000000..063df7a
--- /dev/null
+++ b/src/provisioningserver/boot/tests/test_s390x_partition.py
@@ -0,0 +1,166 @@
+# Copyright 2021 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Tests for `provisioningserver.boot.s390x`."""
+
+
+import random
+import re
+
+from testtools.matchers import MatchesAll, MatchesRegex
+
+from maastesting.factory import factory
+from maastesting.testcase import MAASTestCase
+from provisioningserver.boot import s390x_partition as s390x_partition_module
+from provisioningserver.boot.s390x import format_bootif
+from provisioningserver.boot.s390x_partition import S390XPartitionBootMethod
+from provisioningserver.boot.tftppath import compose_image_path
+from provisioningserver.tests.test_kernel_opts import make_kernel_parameters
+
+
+class TestS390XPartitionBootMethod(MAASTestCase):
+    def test_settings(self):
+        s390x_partition = S390XPartitionBootMethod()
+
+        self.assertEqual("s390x_partition", s390x_partition.name)
+        self.assertEqual("s390x_partition", s390x_partition.bios_boot_method)
+        self.assertEqual("s390x_partition", s390x_partition.template_subdir)
+        self.assertEqual(
+            "s390x_partition/maas", s390x_partition.bootloader_path
+        )
+        self.assertEqual("00:20", s390x_partition.arch_octet)
+        self.assertIsNone(s390x_partition.user_class)
+
+    def test_match_path_matches(self):
+        s390x_partition = S390XPartitionBootMethod()
+        mac = factory.make_mac_address()
+        self.patch(s390x_partition_module, "get_remote_mac").return_value = mac
+
+        self.assertEqual(
+            {
+                "arch": "s390x",
+                "mac": mac,
+            },
+            s390x_partition.match_path(
+                factory.make_name("backend"),
+                s390x_partition.bootloader_path.encode(),
+            ),
+        )
+
+    def test_match_path_doesnt_match(self):
+        s390x_partition = S390XPartitionBootMethod()
+
+        self.assertIsNone(
+            s390x_partition.match_path(
+                factory.make_name("backend"), factory.make_name("path")
+            )
+        )
+
+    def test_get_reader_ephemeral(self):
+        s390x_partition = S390XPartitionBootMethod()
+        mac = factory.make_mac_address()
+        params = make_kernel_parameters(
+            self,
+            arch="s390x",
+            purpose=random.choice(
+                ["commissioning", "enlist", "install", "xinstall"]
+            ),
+        )
+
+        output = s390x_partition.get_reader(None, params, mac)
+        output = output.read(output.size).decode()
+        image_dir = compose_image_path(
+            osystem=params.osystem,
+            arch=params.arch,
+            subarch=params.subarch,
+            release=params.release,
+            label=params.label,
+        )
+
+        self.assertThat(
+            output,
+            MatchesAll(
+                MatchesRegex(
+                    r".*^\s+kernel=%s/%s$"
+                    % (re.escape(image_dir), params.kernel),
+                    re.MULTILINE | re.DOTALL,
+                ),
+                MatchesRegex(
+                    r".*^\s+initrd=%s/%s$"
+                    % (re.escape(image_dir), params.initrd),
+                    re.MULTILINE | re.DOTALL,
+                ),
+                MatchesRegex(
+                    r".*^\s+append=.*BOOTIF=%s$" % format_bootif(mac),
+                    re.MULTILINE | re.DOTALL,
+                ),
+            ),
+        )
+
+    def test_get_reader_ephemeral_no_mac(self):
+        s390x_partition = S390XPartitionBootMethod()
+        params = make_kernel_parameters(
+            self,
+            arch="s390x",
+            purpose=random.choice(
+                ["commissioning", "enlist", "install", "xinstall"]
+            ),
+        )
+
+        output = s390x_partition.get_reader(None, params)
+        output = output.read(output.size).decode()
+        image_dir = compose_image_path(
+            osystem=params.osystem,
+            arch=params.arch,
+            subarch=params.subarch,
+            release=params.release,
+            label=params.label,
+        )
+
+        self.assertThat(
+            output,
+            MatchesAll(
+                MatchesRegex(
+                    r".*^\s+kernel=%s/%s$"
+                    % (re.escape(image_dir), params.kernel),
+                    re.MULTILINE | re.DOTALL,
+                ),
+                MatchesRegex(
+                    r".*^\s+initrd=%s/%s$"
+                    % (re.escape(image_dir), params.initrd),
+                    re.MULTILINE | re.DOTALL,
+                ),
+                MatchesRegex(r".*^\s+append=.*$", re.MULTILINE | re.DOTALL),
+            ),
+        )
+
+    def test_get_reader_poweroff(self):
+        s390x_partition = S390XPartitionBootMethod()
+        mac = factory.make_mac_address()
+        params = make_kernel_parameters(
+            self, arch="s390x", purpose=random.choice(["local", "poweroff"])
+        )
+
+        output = s390x_partition.get_reader(None, params, mac)
+        output = [
+            line
+            for line in output.read(output.size).decode().splitlines()
+            if line.split("#", 1)[0].strip()
+        ]
+
+        self.assertEqual([], output)
+
+    def test_get_reader_poweroff_no_mac(self):
+        s390x_partition = S390XPartitionBootMethod()
+        params = make_kernel_parameters(
+            self, arch="s390x", purpose=random.choice(["local", "poweroff"])
+        )
+
+        output = s390x_partition.get_reader(None, params)
+        output = [
+            line
+            for line in output.read(output.size).decode().splitlines()
+            if line.split("#", 1)[0].strip()
+        ]
+
+        self.assertEqual([], output)
diff --git a/src/provisioningserver/templates/s390x_partition/config.commissioning.s390x.template b/src/provisioningserver/templates/s390x_partition/config.commissioning.s390x.template
new file mode 100644
index 0000000..d6a4d5a
--- /dev/null
+++ b/src/provisioningserver/templates/s390x_partition/config.commissioning.s390x.template
@@ -0,0 +1,7 @@
+PROMPT 0
+DEFAULT netboot
+TIMEOUT 0
+LABEL netboot
+  kernel={{kernel_params | kernel_path }}
+  initrd={{kernel_params | initrd_path }}
+  append={{kernel_params | kernel_command}}
diff --git a/src/provisioningserver/templates/s390x_partition/config.enlist.s390x.template b/src/provisioningserver/templates/s390x_partition/config.enlist.s390x.template
new file mode 120000
index 0000000..0b09e93
--- /dev/null
+++ b/src/provisioningserver/templates/s390x_partition/config.enlist.s390x.template
@@ -0,0 +1 @@
+config.commissioning.s390x.template
\ No newline at end of file
diff --git a/src/provisioningserver/templates/s390x_partition/config.install.s390x.template b/src/provisioningserver/templates/s390x_partition/config.install.s390x.template
new file mode 120000
index 0000000..0b09e93
--- /dev/null
+++ b/src/provisioningserver/templates/s390x_partition/config.install.s390x.template
@@ -0,0 +1 @@
+config.commissioning.s390x.template
\ No newline at end of file
diff --git a/src/provisioningserver/templates/s390x_partition/config.local.s390x.template b/src/provisioningserver/templates/s390x_partition/config.local.s390x.template
new file mode 100644
index 0000000..a35672e
--- /dev/null
+++ b/src/provisioningserver/templates/s390x_partition/config.local.s390x.template
@@ -0,0 +1,2 @@
+# S390X partition must be configured via the HMC to boot locally
+# No config will cause the partition to turn off.
diff --git a/src/provisioningserver/templates/s390x_partition/config.poweroff.s390x.template b/src/provisioningserver/templates/s390x_partition/config.poweroff.s390x.template
new file mode 100644
index 0000000..4541aca
--- /dev/null
+++ b/src/provisioningserver/templates/s390x_partition/config.poweroff.s390x.template
@@ -0,0 +1 @@
+# Sending no config turns off a S390X partition
diff --git a/src/provisioningserver/templates/s390x_partition/config.xinstall.s390x.template b/src/provisioningserver/templates/s390x_partition/config.xinstall.s390x.template
new file mode 120000
index 0000000..0b09e93
--- /dev/null
+++ b/src/provisioningserver/templates/s390x_partition/config.xinstall.s390x.template
@@ -0,0 +1 @@
+config.commissioning.s390x.template
\ No newline at end of file