Merge ~ltrager/maas:vmfs_preseed into maas:master

[Lee Trager]

Modify the storage preseed for VMware ESXi datastores.

[MAAS Lander (maas-lander)]

UNIT TESTS
-b vmfs_preseed lp:~ltrager/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED
LOG: http://maas-ci-jenkins.internal:8080/job/maas/job/branch-tester/5296/console
COMMIT: b35ab37c66b3340c9df371c4d43a3ed264a56871

review: Needs Fixing

63b5c00... by Lee Trager on 2019-03-10

Fix query count

45029bd... by Lee Trager on 2019-03-10

Merge branch 'vmfs_layout' into vmfs_preseed

[MAAS Lander (maas-lander)]

UNIT TESTS
-b vmfs_preseed lp:~ltrager/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: SUCCESS
COMMIT: 45029bd8c921cde741a966c298f260eac457d758

review: Approve

5a167a2... by Lee Trager on 2019-03-11

Set tags to empty list

2067784... by Lee Trager on 2019-03-11

Merge branch 'vmfs_layout' into vmfs_preseed

[MAAS Lander (maas-lander)]

UNIT TESTS
-b vmfs_preseed lp:~ltrager/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED
LOG: http://maas-ci-jenkins.internal:8080/job/maas/job/branch-tester/5300/console
COMMIT: 20677840670b2033075cb37c490245d772ac813b

review: Needs Fixing

956b9fd... by Lee Trager on 2019-03-11

Cleanup comments

aa0aa27... by Lee Trager on 2019-03-11

Merge branch 'vmfs_layout' into vmfs_preseed

6b69ad7... by Lee Trager on 2019-03-11

node.get_osystem() -> node.osystem

[MAAS Lander (maas-lander)]

UNIT TESTS
-b vmfs_preseed lp:~ltrager/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED
LOG: http://maas-ci-jenkins.internal:8080/job/maas/job/branch-tester/5312/console
COMMIT: 6b69ad720b4d9774c0c03b3faadf078ce7100a64

review: Needs Fixing

[Alberto Donato (ack)]

lgtm

review: Approve

[Blake Rouse (blake-rouse)]

No blockers just 2 comments.

review: Approve

1e75131... by Lee Trager on 2019-03-12

Merge branch 'master' into vmfs_layout

0a12c31... by Lee Trager on 2019-03-12

Use bulk_create

b6f090d... by Lee Trager on 2019-03-12

Merge branch 'vmfs_layout' into vmfs_preseed

eba6787... by Lee Trager on 2019-03-12

Set created, and updated time

bc1ef7d... by Lee Trager on 2019-03-12

Merge branch 'vmfs_layout' into vmfs_preseed

875ec3b... by Lee Trager on 2019-03-12

Fix comments

[MAAS Lander (maas-lander)]

UNIT TESTS
-b vmfs_preseed lp:~ltrager/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED
LOG: http://maas-ci-jenkins.internal:8080/job/maas/job/branch-tester/5331/console
COMMIT: 875ec3bbd43f814cbafbc8f89be8eb3980c8a9d1

review: Needs Fixing

35e6240... by Lee Trager on 2019-03-13

Merge branch 'master' into vmfs_layout

fd76dc3... by Lee Trager on 2019-03-13

ack fix

84e3797... by Lee Trager on 2019-03-13

Merge branch 'vmfs_layout' into vmfs_preseed

[MAAS Lander (maas-lander)]

UNIT TESTS
-b vmfs_preseed lp:~ltrager/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED
LOG: http://maas-ci-jenkins.internal:8080/job/maas/job/branch-tester/5345/console
COMMIT: 84e37975533e3d7d5ad68737c3f09a0dc9f0cca3

review: Needs Fixing

c1f64a2... by Lee Trager on 2019-03-15

Fix lint and order VMFS6 devices

diff --git a/src/maasserver/models/partition.py b/src/maasserver/models/partition.py
index 44b93f2..2bd2ad7 100644
--- a/src/maasserver/models/partition.py
+++ b/src/maasserver/models/partition.py
@@ -216,9 +216,15 @@ class Partition(CleanSave, TimestampedModel):
             if (arch == "ppc64el" and block_device.id == boot_disk.id):
                 return idx + 2
             elif arch == "amd64" and bios_boot_method != "uefi":
-                # Delay the `type` check because it can cause a query. Only
-                # physical block devices get the bios_grub partition.
-                if block_device.type == 'physical':
+                if node.osystem == 'esxi':
+                    # VMware ESXi is a DD image but MAAS allows partitions to
+                    # be added to the end of the disk as well as resize the
+                    # datastore partition. The EFI partition is already in the
+                    # image so there is no reason to account for it.
+                    return idx + 1
+                elif block_device.type == 'physical':
+                    # Delay the `type` check because it can cause a query. Only
+                    # physical block devices get the bios_grub partition.
                     return idx + 2
                 else:
                     return idx + 1
diff --git a/src/maasserver/preseed_storage.py b/src/maasserver/preseed_storage.py
index b7daefd..f376f52 100644
--- a/src/maasserver/preseed_storage.py
+++ b/src/maasserver/preseed_storage.py
@@ -1,4 +1,4 @@
-# Copyright 2015-2016 Canonical Ltd.  This software is licensed under the
+# Copyright 2015-2019 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Preseed generation for curtin storage."""
@@ -48,6 +48,7 @@ class CurtinStorageGenerator:
             "lvm_partition": [],
             "raid": [],
             "bcache": [],
+            "vmfs": [],
         }
 
     def generate(self):
@@ -66,6 +67,7 @@ class CurtinStorageGenerator:
         self._generate_logical_volume_operations()
         self._generate_raid_operations()
         self._generate_bcache_operations()
+        self._generate_vmfs_operations()
         self._generate_partition_operations()
         self._generate_format_operations()
 
@@ -110,6 +112,8 @@ class CurtinStorageGenerator:
                     self.operations["raid"].append(filesystem_group)
                 elif filesystem_group.is_bcache():
                     self.operations["bcache"].append(filesystem_group)
+                elif filesystem_group.is_vmfs():
+                    self.operations["vmfs"].append(filesystem_group)
                 else:
                     raise ValueError(
                         "Unknown filesystem group type: %s" % (
@@ -259,10 +263,12 @@ class CurtinStorageGenerator:
                 node_arch == "ppc64el" and
                 bios_boot_method in ("uefi", "powernv", "powerkvm"))
 
-        # always add a boot partition for GPT without UEFI
+        # always add a boot partition for GPT without UEFI. ESXi doesn't
+        # need a partition added as one is already in the DD format.
         add_bios_grub_partition = (
             disk_operation.get("ptable") == "gpt" and
-            node_arch == "amd64" and bios_boot_method != "uefi")
+            node_arch == "amd64" and bios_boot_method != "uefi" and
+            self.node.osystem != "esxi")
 
         # Set this disk to be the grub device if it's the boot disk and doesn't
         # require a prep partition. When a prep partition is required grub
@@ -525,6 +531,24 @@ class CurtinStorageGenerator:
             bcache_operation["ptable"] = self._get_ptable_type(partition_table)
         self.storage_config.append(bcache_operation)
 
+    def _generate_vmfs_operations(self):
+        """Generate all vmfs operations."""
+        for vmfs in self.operations["vmfs"]:
+            self.storage_config.append({
+                "id": vmfs.name,
+                "name": vmfs.name,
+                "type": "vmfs6",
+                "devices": sorted([
+                    fs.get_parent().name for fs in vmfs.filesystems.all()]),
+            })
+
+    def _reorder_devices(self, ids_above, operation):
+        for device in operation["devices"]:
+            if device not in ids_above:
+                self._reorder_operation(operation, device)
+                return True
+        return False
+
     def _order_config_dependency(self):
         """Re-order the storage config so dependencies appear before
         dependents."""
@@ -548,13 +572,7 @@ class CurtinStorageGenerator:
                         self._reorder_operation(operation, volume)
                         break
                 elif operation_type == "lvm_volgroup":
-                    exit_early = False
-                    for device in operation["devices"]:
-                        if device not in ids_above:
-                            self._reorder_operation(operation, device)
-                            exit_early = True
-                            break
-                    if exit_early:
+                    if self._reorder_devices(ids_above, operation):
                         break
                 elif operation_type == "lvm_partition":
                     volgroup = operation["volgroup"]
@@ -562,14 +580,9 @@ class CurtinStorageGenerator:
                         self._reorder_operation(operation, volgroup)
                         break
                 elif operation_type == "raid":
-                    exit_early = False
-                    for device in operation["devices"]:
-                        if device not in ids_above:
-                            self._reorder_operation(operation, device)
-                            exit_early = True
-                            break
-                    if exit_early:
+                    if self._reorder_devices(ids_above, operation):
                         break
+                    exit_early = False
                     for device in operation["spare_devices"]:
                         if device not in ids_above:
                             self._reorder_operation(operation, device)
@@ -586,6 +599,9 @@ class CurtinStorageGenerator:
                     if cache_device not in ids_above:
                         self._reorder_operation(operation, cache_device)
                         break
+                elif operation_type == "vmfs6":
+                    if self._reorder_devices(ids_above, operation):
+                        break
                 else:
                     raise ValueError(
                         "Unknown operation type: %s" % operation_type)
diff --git a/src/maasserver/tests/test_preseed_storage.py b/src/maasserver/tests/test_preseed_storage.py
index 390ff4c..819069c 100644
--- a/src/maasserver/tests/test_preseed_storage.py
+++ b/src/maasserver/tests/test_preseed_storage.py
@@ -1,4 +1,4 @@
-# Copyright 2015-2016 Canonical Ltd.  This software is licensed under the
+# Copyright 2015-2019 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Test `maasserver.preseed_storage`."""
@@ -16,9 +16,11 @@ from maasserver.enum import (
     NODE_STATUS,
     PARTITION_TABLE_TYPE,
 )
+from maasserver.models import Filesystem
 from maasserver.models.filesystemgroup import (
     Bcache,
     RAID,
+    VMFS,
     VolumeGroup,
 )
 from maasserver.models.partitiontable import (
@@ -27,6 +29,7 @@ from maasserver.models.partitiontable import (
     PREP_PARTITION_SIZE,
 )
 from maasserver.preseed_storage import compose_curtin_storage_config
+from maasserver.storage_layouts import VMFS6Layout
 from maasserver.testing.factory import factory
 from maasserver.testing.testcase import MAASServerTestCase
 from testtools.content import text_content
@@ -2069,3 +2072,157 @@ class TestBootableRaidLayoutGPTWithPartition(
         node._create_acquired_filesystems()
         config = compose_curtin_storage_config(node)
         self.assertStorageConfig(self.STORAGE_CONFIG, config)
+
+
+class TestVMFS(MAASServerTestCase, AssertStorageConfigMixin):
+
+    STORAGE_CONFIG = dedent("""\
+        config:
+          - id: sda
+            model: vendor
+            name: sda
+            ptable: gpt
+            serial: serial-a
+            type: disk
+            wipe: superblock
+            grub_device: true
+          - id: sdb
+            model: vendor
+            name: sdb
+            ptable: gpt
+            serial: serial-b
+            type: disk
+            wipe: superblock
+          - id: sdc
+            model: vendor
+            name: sdc
+            ptable: gpt
+            serial: serial-c
+            type: disk
+            wipe: superblock
+          - id: sda-part1
+            name: sda-part1
+            device: sda
+            number: 1
+            type: partition
+            size: 3145728B
+            uuid: 7f79841c-9f57-4ab7-ada2-b2774e3908a3
+            wipe: superblock
+            flag: boot
+          - id: sda-part2
+            name: sda-part2
+            device: sda
+            number: 2
+            type: partition
+            size: 4294967296B
+            uuid: b1a04342-60a2-47ca-8773-32b3af24c73e
+            wipe: superblock
+          - id: sda-part3
+            name: sda-part3
+            device: sda
+            number: 3
+            type: partition
+            size: 99451142144B
+            uuid: 54e698a9-e15d-409b-9e0a-2791374f6c12
+            wipe: superblock
+          - id: sda-part4
+            name: sda-part4
+            device: sda
+            number: 4
+            type: partition
+            size: 261095424B
+            uuid: af8404ca-5d62-402c-8f11-604dc0ba30a6
+            wipe: superblock
+          - id: sda-part5
+            name: sda-part5
+            device: sda
+            number: 5
+            type: partition
+            size: 261095424B
+            uuid: 34949af8-dc2f-4f9a-b81a-f26e12499f7b
+            wipe: superblock
+          - id: sda-part6
+            name: sda-part6
+            device: sda
+            number: 6
+            type: partition
+            size: 114294784B
+            uuid: e29dd323-671e-4e8f-81fe-f1f71492f231
+            wipe: superblock
+          - id: sda-part7
+            name: sda-part7
+            device: sda
+            number: 7
+            type: partition
+            size: 298844160B
+            uuid: 817ddd58-f5af-4d9a-8896-c20c06cf0f7f
+            wipe: superblock
+          - id: sda-part8
+            name: sda-part8
+            device: sda
+            number: 8
+            type: partition
+            size: 2684354560B
+            uuid: b5a745b6-7247-4397-a9a9-d484de69e35c
+            wipe: superblock
+          - id: sdb-part1
+            name: sdb-part1
+            device: sdb
+            number: 1
+            type: partition
+            offset: 4194304B
+            size: 107365793792B
+            uuid: e2565df0-8ec8-4d0e-88a6-92836fce3f58
+            wipe: superblock
+          - id: datastore1
+            name: datastore1
+            type: vmfs6
+            devices:
+              - sda-part3
+              - sdb-part1
+          - id: sdc-part1
+            name: sdc-part1
+            device: sdc
+            number: 1
+            type: partition
+            offset: 4194304B
+            size: 107365793792B
+            uuid: c803220b-7533-4276-a9b4-bfebb68813d0
+            wipe: superblock
+          - id: datastore2
+            name: datastore2
+            type: vmfs6
+            devices:
+              - sdc-part1
+    """)
+
+    def test__renders_expected_output(self):
+        node = factory.make_Node(
+            status=NODE_STATUS.ALLOCATED, architecture="amd64/generic",
+            osystem='esxi', distro_series='6.7', with_boot_disk=False)
+        boot_disk = factory.make_PhysicalBlockDevice(
+            node=node, size=100 * 1024 ** 3, name='sda',
+            model='vendor', serial='serial-a')
+        layout = VMFS6Layout(node)
+        layout.configure()
+        extra_disk = factory.make_PhysicalBlockDevice(
+            node=node, size=100 * 1024 ** 3, name='sdb', model='vendor',
+            serial='serial-b')
+        for vmfs_part in boot_disk.get_partitiontable().partitions.all():
+            if 'part3' in vmfs_part.name:
+                break
+        vmfs = vmfs_part.get_effective_filesystem().filesystem_group
+        partition = extra_disk.create_partition()
+        Filesystem.objects.create(
+            fstype=FILESYSTEM_TYPE.VMFS6, partition=partition,
+            filesystem_group=vmfs)
+        extra_datastore_disk = factory.make_PhysicalBlockDevice(
+            node=node, size=100 * 1024 ** 3, name='sdc', model='vendor',
+            serial='serial-c')
+        extra_datastore_part = extra_datastore_disk.create_partition()
+        VMFS.objects.create_vmfs(
+            name="datastore2", partitions=[extra_datastore_part])
+
+        node._create_acquired_filesystems()
+        config = compose_curtin_storage_config(node)
+        self.assertStorageConfig(self.STORAGE_CONFIG, config, True)