Merge ~ltrager/maas:vmfs_model into maas:master

[Lee Trager]

Add VMFS Filesystemgroup

[Newell Jensen (newell-jensen)]

Looks good. Simple and clean. Couple inline suggestions.

review: Approve

[MAAS Lander (maas-lander)]

UNIT TESTS
-b vmfs_model lp:~ltrager/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: SUCCESS
COMMIT: b354e2ba5d647f95a4fd312baa2977c95fddb228

review: Approve

796e6a5... by Lee Trager on 2019-02-21

Merge branch 'master' into vmfs_model

[MAAS Lander (maas-lander)]

UNIT TESTS
-b vmfs_model lp:~ltrager/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: SUCCESS
COMMIT: 796e6a5d8c7e107f4b5460d4ea614854c3ae86bc

review: Approve

215a78c... by Lee Trager on 2019-02-21

Merge branch 'master' into vmfs_model

f63af7c... by Lee Trager on 2019-02-22

Merge branch 'master' into vmfs_model

[MAAS Lander (maas-lander)]

UNIT TESTS
-b vmfs_model lp:~ltrager/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED
LOG: http://maas-ci-jenkins.internal:8080/job/maas/job/branch-tester/5176/console
COMMIT: f63af7c7ef8f3b9a1642fd8066b1104375725453

review: Needs Fixing

07766ac... by Lee Trager on 2019-02-27

Merge branch 'master' into vmfs_model

[MAAS Lander (maas-lander)]

UNIT TESTS
-b vmfs_model lp:~ltrager/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: SUCCESS
COMMIT: 07766ac4c207213545b4805022bc4ba49d4d026b

review: Approve

df46675... by Lee Trager on 2019-03-07

Merge branch 'master' into vmfs_model

2c2bfd2... by Lee Trager on 2019-03-07

Reorder migration

3514853... by Lee Trager on 2019-03-07

Add migration

[MAAS Lander (maas-lander)]

UNIT TESTS
-b vmfs_model lp:~ltrager/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: SUCCESS
COMMIT: 3514853a452ea7453cbb54f504499508d7501728

review: Approve

diff --git a/src/maasserver/enum.py b/src/maasserver/enum.py
index b50e61c..ddd2948 100644
--- a/src/maasserver/enum.py
+++ b/src/maasserver/enum.py
@@ -1,4 +1,4 @@
-# Copyright 2012-2018 Canonical Ltd.  This software is licensed under the
+# Copyright 2012-2019 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Enumerations meaningful to the maasserver application."""
@@ -511,6 +511,9 @@ class FILESYSTEM_TYPE:
     #: ZFS
     ZFSROOT = "zfsroot"
 
+    #: VMFS6
+    VMFS6 = "vmfs6"
+
 
 # Django choices for FILESYSTEM_TYPE: sequence of tuples (key, UI
 # representation).
@@ -532,6 +535,7 @@ FILESYSTEM_TYPE_CHOICES = (
     (FILESYSTEM_TYPE.TMPFS, "tmpfs"),
     (FILESYSTEM_TYPE.BTRFS, "btrfs"),
     (FILESYSTEM_TYPE.ZFSROOT, "zfsroot"),
+    (FILESYSTEM_TYPE.VMFS6, "vmfs6"),
     )
 
 
@@ -581,6 +585,9 @@ class FILESYSTEM_GROUP_TYPE:
     #: Bcache
     BCACHE = 'bcache'
 
+    #: VMFS6
+    VMFS6 = 'vmfs6'
+
 
 FILESYSTEM_GROUP_RAID_TYPES = [
     FILESYSTEM_GROUP_TYPE.RAID_0,
@@ -605,6 +612,7 @@ FILESYSTEM_GROUP_RAID_TYPE_CHOICES = (
 FILESYSTEM_GROUP_TYPE_CHOICES = FILESYSTEM_GROUP_RAID_TYPE_CHOICES + (
     (FILESYSTEM_GROUP_TYPE.LVM_VG, "LVM VG"),
     (FILESYSTEM_GROUP_TYPE.BCACHE, "Bcache"),
+    (FILESYSTEM_GROUP_TYPE.VMFS6, "VMFS6"),
     )
 
 
diff --git a/src/maasserver/migrations/builtin/maasserver/0185_vmfs6.py b/src/maasserver/migrations/builtin/maasserver/0185_vmfs6.py
new file mode 100644
index 0000000..f10c145
--- /dev/null
+++ b/src/maasserver/migrations/builtin/maasserver/0185_vmfs6.py
@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.11 on 2019-03-07 08:41
+from __future__ import unicode_literals
+
+from django.db import (
+    migrations,
+    models,
+)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('maasserver', '0184_add_ephemeral_deploy_setting_to_node'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='VMFS',
+            fields=[
+            ],
+            options={
+                'proxy': True,
+                'indexes': [],
+            },
+            bases=('maasserver.filesystemgroup',),
+        ),
+        migrations.AlterField(
+            model_name='filesystem',
+            name='fstype',
+            field=models.CharField(choices=[('ext2', 'ext2'), ('ext4', 'ext4'), ('xfs', 'xfs'), ('fat32', 'fat32'), ('vfat', 'vfat'), ('lvm-pv', 'lvm'), ('raid', 'raid'), ('raid-spare', 'raid-spare'), ('bcache-cache', 'bcache-cache'), ('bcache-backing', 'bcache-backing'), ('swap', 'swap'), ('ramfs', 'ramfs'), ('tmpfs', 'tmpfs'), ('btrfs', 'btrfs'), ('zfsroot', 'zfsroot'), ('vmfs6', 'vmfs6')], default='ext4', max_length=20),
+        ),
+        migrations.AlterField(
+            model_name='filesystemgroup',
+            name='group_type',
+            field=models.CharField(choices=[('raid-0', 'RAID 0'), ('raid-1', 'RAID 1'), ('raid-5', 'RAID 5'), ('raid-6', 'RAID 6'), ('raid-10', 'RAID 10'), ('lvm-vg', 'LVM VG'), ('bcache', 'Bcache'), ('vmfs6', 'VMFS6')], max_length=20),
+        ),
+    ]
diff --git a/src/maasserver/models/__init__.py b/src/maasserver/models/__init__.py
index 6abec61..b330257 100644
--- a/src/maasserver/models/__init__.py
+++ b/src/maasserver/models/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2012-2018 Canonical Ltd.  This software is licensed under the
+# Copyright 2012-2019 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Model export and helpers for maasserver."""
@@ -87,6 +87,7 @@ __all__ = [
     'VLAN',
     'VLANInterface',
     'VolumeGroup',
+    'VMFS',
     'Zone',
 ]
 
@@ -131,6 +132,7 @@ from maasserver.models.filesystemgroup import (
     Bcache,
     FilesystemGroup,
     RAID,
+    VMFS,
     VolumeGroup,
 )
 from maasserver.models.globaldefault import GlobalDefault
diff --git a/src/maasserver/models/filesystemgroup.py b/src/maasserver/models/filesystemgroup.py
index 16b644e..38edce7 100644
--- a/src/maasserver/models/filesystemgroup.py
+++ b/src/maasserver/models/filesystemgroup.py
@@ -1,11 +1,15 @@
-# Copyright 2015-2016 Canonical Ltd.  This software is licensed under the
+# Copyright 2015-2019 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Model for a filesystem group. Contains a set of filesystems that create
 a virtual block device. E.g. LVM Volume Group."""
 
 __all__ = [
+    'Bcache',
     'FilesystemGroup',
+    'RAID',
+    'VMFS',
+    'VolumeGroup',
     ]
 
 from uuid import uuid4
@@ -273,6 +277,24 @@ class BcacheManager(BaseFilesystemGroupManager):
         return bcache_filesystem_group
 
 
+class VMFSManager(BaseFilesystemGroupManager):
+    """VMFS filesystem group"""
+
+    extra_filters = {'group_type': FILESYSTEM_GROUP_TYPE.VMFS6}
+
+    def create_vmfs(self, name, partitions, uuid=None):
+        """Create a `VMFS` with the list of block devices and partitions."""
+        # Avoid circular imports.
+        from maasserver.models.filesystem import Filesystem
+        vmfs = self.create(name=name, uuid=uuid)
+        for partition in partitions:
+            Filesystem.objects.create(
+                fstype=FILESYSTEM_TYPE.VMFS6, partition=partition,
+                filesystem_group=vmfs)
+        vmfs.save(force_update=True)
+        return vmfs
+
+
 class FilesystemGroup(CleanSave, TimestampedModel):
     """A filesystem group. Contains a set of filesystems that create
     a virtual block device. E.g. LVM Volume Group.
@@ -350,6 +372,8 @@ class FilesystemGroup(CleanSave, TimestampedModel):
             return self.get_raid_size()
         elif self.is_bcache():
             return self.get_bcache_size()
+        elif self.is_vmfs():
+            return self.get_total_size()
         else:
             return 0
 
@@ -384,6 +408,13 @@ class FilesystemGroup(CleanSave, TimestampedModel):
                 for filesystem in filesystems
                 )
 
+    def get_total_size(self):
+        """Return the size of all filesystems combined."""
+        total = 0
+        for fs in self.filesystems.all():
+            total += fs.get_size()
+        return total
+
     def get_raid_size(self):
         """Size of this RAID.
 
@@ -522,6 +553,10 @@ class FilesystemGroup(CleanSave, TimestampedModel):
         """Return True if `group_type` is BCACHE type."""
         return self.group_type == FILESYSTEM_GROUP_TYPE.BCACHE
 
+    def is_vmfs(self):
+        """Return True if `group_type` is VMFS."""
+        return self.group_type == FILESYSTEM_GROUP_TYPE.VMFS6
+
     def _get_all_fstypes(self, filesystems=None):
         """Return list of all filesystem types attached."""
         # Grab all filesystems so that if the filesystems have been
@@ -702,6 +737,8 @@ class FilesystemGroup(CleanSave, TimestampedModel):
             return "RAID"
         elif self.is_bcache():
             return "Bcache"
+        elif self.is_vmfs():
+            return "VMFS"
         else:
             raise ValueError("Unknown group_type.")
 
@@ -714,6 +751,8 @@ class FilesystemGroup(CleanSave, TimestampedModel):
             return "md"
         elif self.is_bcache():
             return "bcache"
+        elif self.is_vmfs():
+            return "vmfs"
         else:
             raise ValidationError("Unknown group_type.")
 
@@ -729,6 +768,9 @@ class FilesystemGroup(CleanSave, TimestampedModel):
         elif self.is_bcache():
             # Bcache uses the block_size of the backing device.
             return self.get_bcache_backing_filesystem().get_block_size()
+        elif self.is_vmfs():
+            # By default VMFS uses a 1MB block size.
+            return 1024
         else:
             raise ValidationError("Unknown group_type.")
 
@@ -913,3 +955,16 @@ class Bcache(FilesystemGroup):
     def __init__(self, *args, **kwargs):
         super(Bcache, self).__init__(
             group_type=FILESYSTEM_GROUP_TYPE.BCACHE, *args, **kwargs)
+
+
+class VMFS(FilesystemGroup):
+    """A VMFS."""
+
+    objects = VMFSManager()
+
+    class Meta(DefaultMeta):
+        proxy = True
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(
+            group_type=FILESYSTEM_GROUP_TYPE.VMFS6, *args, **kwargs)
diff --git a/src/maasserver/models/tests/test_filesystemgroup.py b/src/maasserver/models/tests/test_filesystemgroup.py
index 9d2d0e3..c815abf 100644
--- a/src/maasserver/models/tests/test_filesystemgroup.py
+++ b/src/maasserver/models/tests/test_filesystemgroup.py
@@ -1,4 +1,4 @@
-# Copyright 2015-2016 Canonical Ltd.  This software is licensed under the
+# Copyright 2015-2019 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Tests for `FilesystemGroup`."""
@@ -827,6 +827,17 @@ class TestFilesystemGroup(MAASServerTestCase):
             filesystems=filesystems)
         self.assertEqual(backing_size, fsgroup.get_size())
 
+    def test_get_size_returns_total_size_with_vmfs(self):
+        vmfs = factory.make_VMFS()
+        self.assertEquals(vmfs.get_total_size(), vmfs.get_size())
+
+    def test_get_total_size(self):
+        vmfs = factory.make_VMFS()
+        size = 0
+        for fs in vmfs.filesystems.all():
+            size += fs.get_size()
+        self.assertEquals(size, vmfs.get_total_size())
+
     def test_is_lvm_returns_true_when_LVM_VG(self):
         fsgroup = FilesystemGroup(group_type=FILESYSTEM_GROUP_TYPE.LVM_VG)
         self.assertTrue(fsgroup.is_lvm())
@@ -863,6 +874,10 @@ class TestFilesystemGroup(MAASServerTestCase):
                 FILESYSTEM_GROUP_TYPE, but_not=FILESYSTEM_GROUP_TYPE.BCACHE))
         self.assertFalse(fsgroup.is_bcache())
 
+    def test_is_vmfs(self):
+        vmfs = factory.make_VMFS()
+        self.assertTrue(vmfs.is_vmfs())
+
     def test_can_save_new_filesystem_group_without_filesystems(self):
         fsgroup = FilesystemGroup(
             group_type=FILESYSTEM_GROUP_TYPE.LVM_VG,
@@ -1419,6 +1434,10 @@ class TestFilesystemGroupGetNiceName(MAASServerTestCase):
             "group_type": FILESYSTEM_GROUP_TYPE.BCACHE,
             "name": "Bcache",
             }),
+        (FILESYSTEM_GROUP_TYPE.VMFS6, {
+            "group_type": FILESYSTEM_GROUP_TYPE.VMFS6,
+            "name": "VMFS",
+            }),
     ]
 
     def test__returns_prefix(self):
@@ -1459,6 +1478,10 @@ class TestFilesystemGroupGetNamePrefix(MAASServerTestCase):
             "group_type": FILESYSTEM_GROUP_TYPE.BCACHE,
             "prefix": "bcache",
             }),
+        (FILESYSTEM_GROUP_TYPE.VMFS6, {
+            "group_type": FILESYSTEM_GROUP_TYPE.VMFS6,
+            "prefix": "vmfs",
+            }),
     ]
 
     def test__returns_prefix(self):
@@ -1495,6 +1518,10 @@ class TestFilesystemGroupGetVirtualBlockDeviceBlockSize(MAASServerTestCase):
             "group_type": FILESYSTEM_GROUP_TYPE.RAID_10,
             "block_size": 512,
             }),
+        (FILESYSTEM_GROUP_TYPE.VMFS6, {
+            "group_type": FILESYSTEM_GROUP_TYPE.VMFS6,
+            "block_size": 1024,
+            }),
         # For BCACHE see
         # `test_get_virtual_block_device_block_size_returns_backing_for_bc`
         # above.
diff --git a/src/maasserver/testing/factory.py b/src/maasserver/testing/factory.py
index cdc72ae..80427c5 100644
--- a/src/maasserver/testing/factory.py
+++ b/src/maasserver/testing/factory.py
@@ -2206,6 +2206,12 @@ class Factory(maastesting.factory.Factory):
                     fstype=FILESYSTEM_TYPE.BCACHE_BACKING,
                     block_device=backing_block_device)
                 group.filesystems.add(backing_filesystem)
+            elif group_type == FILESYSTEM_GROUP_TYPE.VMFS6:
+                for _ in range(2):
+                    partition = self.make_Partition(node=node)
+                    self.make_Filesystem(
+                        fstype=FILESYSTEM_TYPE.VMFS6, partition=partition,
+                        filesystem_group=group)
         else:
             for filesystem in filesystems:
                 group.filesystems.add(filesystem)
@@ -2221,6 +2227,10 @@ class Factory(maastesting.factory.Factory):
         filesystem_group = self.make_FilesystemGroup(*args, **kwargs)
         return VolumeGroup.objects.get(id=filesystem_group.id)
 
+    def make_VMFS(self, *args, **kwargs):
+        return self.make_FilesystemGroup(
+            *args, group_type=FILESYSTEM_GROUP_TYPE.VMFS6, **kwargs)
+
     def make_VirtualBlockDevice(
             self, name=None, size=None, block_size=None,
             tags=None, uuid=None, filesystem_group=None, node=None):