Merge lp:~lamont/maas/raid10 into lp:~maas-committers/maas/trunk

[LaMont Jones]

Add RAID 10 support to storage.

Add RAID 10 support to storage.

[Blake Rouse (blake-rouse)]

At quick glance looks to be missing unit tests for the JS.

review: Needs Fixing

[LaMont Jones (lamont)]

> At quick glance looks to be missing unit tests for the JS.

Good catch. Added those, and updated them to fail if someone adds another raid type without fixing them.

[Blake Rouse (blake-rouse)]

Looks really good!

review: Approve

=== modified file 'src/maasserver/api/tests/test_raid.py'
--- src/maasserver/api/tests/test_raid.py	2015-09-24 16:22:12 +0000
+++ src/maasserver/api/tests/test_raid.py	2015-10-29 13:27:52 +0000
@@ -97,6 +97,8 @@
                 node=node, group_type=FILESYSTEM_GROUP_TYPE.RAID_5),
             factory.make_FilesystemGroup(
                 node=node, group_type=FILESYSTEM_GROUP_TYPE.RAID_6),
+            factory.make_FilesystemGroup(
+                node=node, group_type=FILESYSTEM_GROUP_TYPE.RAID_10),
         ]
         # Not RAID. Should not be in the output.
         for _ in range(3):
@@ -419,7 +421,7 @@
         bds = [
             factory.make_PhysicalBlockDevice(node=node, size=10 * 1000 ** 4)
             for i in range(10)
-        ]
+            ]
         for bd in bds[5:]:
             factory.make_PartitionTable(block_device=bd)
         for bd in bds[5:]:
@@ -446,7 +448,7 @@
 
         parsed_device = json.loads(response.content)
         (parsed_block_devices, parsed_partitions,
-            parsed_block_device_spares, parsed_partition_spares) = (
+         parsed_block_device_spares, parsed_partition_spares) = (
             get_devices_from_raid(parsed_device))
         # Size is equivalent to 6 devices of 9 TB each.
         self.assertEqual(
@@ -457,6 +459,53 @@
         self.assertItemsEqual(spare_devices, parsed_block_device_spares)
         self.assertItemsEqual(spare_partitions, parsed_partition_spares)
 
+    def test_create_raid_10(self):
+        """Checks it's possible to create a RAID 10 using 4 raw
+        devices, 4 partitions, one spare device and one spare partition."""
+        self.become_admin()
+        node = factory.make_Node(status=NODE_STATUS.READY)
+        # Add 10 10TB physical block devices to the node.
+        bds = [
+            factory.make_PhysicalBlockDevice(node=node, size=10 * 1000 ** 4)
+            for i in range(10)
+        ]
+        for bd in bds[5:]:
+            factory.make_PartitionTable(block_device=bd)
+        for bd in bds[5:]:
+            bd.get_partitiontable().add_partition(size=1000 ** 4)
+        large_partitions = [bd.get_partitiontable().add_partition()
+                            for bd in bds[5:]]
+        uuid4 = unicode(uuid.uuid4())
+        uri = get_raid_devices_uri(node)
+        block_devices = [bd.id for bd in bds[1:]
+                         if bd.get_partitiontable() is None]
+        partitions = [lp.id for lp in large_partitions[1:]]
+        spare_devices = [bds[0].id]
+        spare_partitions = [large_partitions[0].id]
+        response = self.client.post(uri, {
+            'name': 'md0',
+            'uuid': uuid4,
+            'level': FILESYSTEM_GROUP_TYPE.RAID_10,
+            'block_devices': block_devices,
+            'partitions': partitions,
+            'spare_devices': spare_devices,
+            'spare_partitions': spare_partitions,
+        })
+        self.assertEqual(httplib.OK, response.status_code, response.content)
+
+        parsed_device = json.loads(response.content)
+        (parsed_block_devices, parsed_partitions,
+            parsed_block_device_spares, parsed_partition_spares) = (
+            get_devices_from_raid(parsed_device))
+        # Size is equivalent to 4 devices of 9 TB each.
+        self.assertEqual(
+            4 * ((9 * 1000 ** 4) - PARTITION_TABLE_EXTRA_SPACE),
+            parsed_device['size'])
+        self.assertItemsEqual(block_devices, parsed_block_devices)
+        self.assertItemsEqual(partitions, parsed_partitions)
+        self.assertItemsEqual(spare_devices, parsed_block_device_spares)
+        self.assertItemsEqual(spare_partitions, parsed_partition_spares)
+
     def test_create_raid_5_with_2_elements_fails(self):
         self.become_admin()
         node = factory.make_Node(status=NODE_STATUS.READY)
@@ -486,13 +535,35 @@
         bds = [
             factory.make_PhysicalBlockDevice(node=node, size=10 * 1000 ** 4)
             for i in range(3)
+            ]
+        uuid4 = unicode(uuid.uuid4())
+        uri = get_raid_devices_uri(node)
+        response = self.client.post(uri, {
+            'name': 'md0',
+            'uuid': uuid4,
+            'level': FILESYSTEM_GROUP_TYPE.RAID_6,
+            'block_devices': [bd.id for bd in bds],
+            'partitions': [],
+            'spare_devices': [],
+            'spare_partitions': [],
+        })
+        self.assertEqual(httplib.BAD_REQUEST, response.status_code,
+                         response.content)
+
+    def test_create_raid_10_with_2_elements_fails(self):
+        self.become_admin()
+        node = factory.make_Node(status=NODE_STATUS.READY)
+        # Add 3 10TB physical block devices to the node.
+        bds = [
+            factory.make_PhysicalBlockDevice(node=node, size=10 * 1000 ** 4)
+            for i in range(2)
         ]
         uuid4 = unicode(uuid.uuid4())
         uri = get_raid_devices_uri(node)
         response = self.client.post(uri, {
             'name': 'md0',
             'uuid': uuid4,
-            'level': FILESYSTEM_GROUP_TYPE.RAID_6,
+            'level': FILESYSTEM_GROUP_TYPE.RAID_10,
             'block_devices': [bd.id for bd in bds],
             'partitions': [],
             'spare_devices': [],
 
=== modified file 'src/maasserver/enum.py'
--- src/maasserver/enum.py	2015-10-04 20:47:36 +0000
+++ src/maasserver/enum.py	2015-10-29 13:27:52 +0000
@@ -463,6 +463,9 @@
     #: RAID level 6
     RAID_6 = 'raid-6'
 
+    #: RAID level 10
+    RAID_10 = 'raid-10'
+
     #: Bcache
     BCACHE = 'bcache'
 
@@ -472,6 +475,7 @@
     FILESYSTEM_GROUP_TYPE.RAID_1,
     FILESYSTEM_GROUP_TYPE.RAID_5,
     FILESYSTEM_GROUP_TYPE.RAID_6,
+    FILESYSTEM_GROUP_TYPE.RAID_10,
     ]
 
 # Django choices for FILESYSTEM_GROUP_RAID_TYPES: sequence of tuples (key, UI
@@ -481,6 +485,7 @@
     (FILESYSTEM_GROUP_TYPE.RAID_1, "RAID 1"),
     (FILESYSTEM_GROUP_TYPE.RAID_5, "RAID 5"),
     (FILESYSTEM_GROUP_TYPE.RAID_6, "RAID 6"),
+    (FILESYSTEM_GROUP_TYPE.RAID_10, "RAID 10"),
     )
 
 # Django choices for FILESYSTEM_GROUP_TYPE: sequence of tuples (key, UI
 
=== modified file 'src/maasserver/models/filesystemgroup.py'
--- src/maasserver/models/filesystemgroup.py	2015-10-09 04:35:47 +0000
+++ src/maasserver/models/filesystemgroup.py	2015-10-29 13:27:52 +0000
@@ -417,6 +417,8 @@
                 return min_size * (num_raid - 1)
             elif self.group_type == FILESYSTEM_GROUP_TYPE.RAID_6:
                 return min_size * (num_raid - 2)
+            elif self.group_type == FILESYSTEM_GROUP_TYPE.RAID_10:
+                return min_size * num_raid / 2
         raise ValidationError("Unknown raid type: %s" % self.group_type)
 
     def get_bcache_backing_filesystem(self):
@@ -583,6 +585,13 @@
                 raise ValidationError(
                     "RAID level 6 must have at least 4 raid devices and "
                     "any number of spares.")
+        elif self.group_type == FILESYSTEM_GROUP_TYPE.RAID_10:
+            # RAID 10 must have at least 4 RAID filesystems, but can have
+            # spares.
+            if num_raid < 3:
+                raise ValidationError(
+                    "RAID level 10 must have at least 3 raid devices and "
+                    "any number of spares.")
         num_raid_invalid = len([
             fstype
             for fstype in fstypes
 
=== modified file 'src/maasserver/models/tests/test_filesystemgroup.py'
--- src/maasserver/models/tests/test_filesystemgroup.py	2015-10-09 04:35:47 +0000
+++ src/maasserver/models/tests/test_filesystemgroup.py	2015-10-29 13:27:52 +0000
@@ -769,6 +769,36 @@
         self.assertEquals(
             small_size * (number_of_raid_devices - 1), fsgroup.get_size())
 
+    def test_get_size_returns_correct_disk_size_for_raid_10(self):
+        node = factory.make_Node()
+        small_size = random.randint(
+            MIN_BLOCK_DEVICE_SIZE, MIN_BLOCK_DEVICE_SIZE ** 2)
+        other_size = random.randint(small_size + 1, small_size + (10 ** 5))
+        number_of_raid_devices = random.randint(3, 9)
+        filesystems = [
+            factory.make_Filesystem(
+                fstype=FILESYSTEM_TYPE.RAID,
+                block_device=factory.make_PhysicalBlockDevice(
+                    node=node, size=small_size)),
+        ]
+        for _ in range(number_of_raid_devices):
+            filesystems.append(
+                factory.make_Filesystem(
+                    fstype=FILESYSTEM_TYPE.RAID,
+                    block_device=factory.make_PhysicalBlockDevice(
+                        node=node, size=other_size)))
+        # Spares are ignored and not taken into calculation.
+        for _ in range(3):
+            filesystems.append(
+                factory.make_Filesystem(
+                    fstype=FILESYSTEM_TYPE.RAID_SPARE,
+                    block_device=factory.make_PhysicalBlockDevice(
+                        node=node, size=other_size)))
+        fsgroup = factory.make_FilesystemGroup(
+            group_type=FILESYSTEM_GROUP_TYPE.RAID_10, filesystems=filesystems)
+        self.assertEquals(
+            small_size * (number_of_raid_devices + 1) / 2, fsgroup.get_size())
+
     def test_get_size_returns_0_if_bcache_without_backing(self):
         fsgroup = FilesystemGroup(group_type=FILESYSTEM_GROUP_TYPE.BCACHE)
         self.assertEquals(0, fsgroup.get_size())
@@ -1071,7 +1101,7 @@
                 fstype=FILESYSTEM_TYPE.RAID,
                 block_device=factory.make_PhysicalBlockDevice(node=node))
             for _ in range(random.randint(1, 3))
-        ]
+            ]
         with ExpectedException(
                 ValidationError,
                 re.escape(
@@ -1088,7 +1118,7 @@
                 fstype=FILESYSTEM_TYPE.RAID,
                 block_device=factory.make_PhysicalBlockDevice(node=node))
             for _ in range(random.randint(4, 10))
-        ]
+            ]
         for _ in range(random.randint(1, 5)):
             filesystems.append(
                 factory.make_Filesystem(
@@ -1099,6 +1129,59 @@
             group_type=FILESYSTEM_GROUP_TYPE.RAID_6,
             filesystems=filesystems)
 
+    def test_cannot_save_raid_10_with_less_than_3_raid_devices(self):
+        node = factory.make_Node()
+        filesystems = [
+            factory.make_Filesystem(
+                fstype=FILESYSTEM_TYPE.RAID,
+                block_device=factory.make_PhysicalBlockDevice(node=node))
+            for _ in range(random.randint(1, 2))
+        ]
+        with ExpectedException(
+                ValidationError,
+                re.escape(
+                    "{'__all__': [u'RAID level 10 must have at least 3 raid "
+                    "devices and any number of spares.']}")):
+            factory.make_FilesystemGroup(
+                group_type=FILESYSTEM_GROUP_TYPE.RAID_10,
+                filesystems=filesystems)
+
+    def test_can_save_raid_10_with_3_raid_devices_and_spares(self):
+        node = factory.make_Node()
+        filesystems = [
+            factory.make_Filesystem(
+                fstype=FILESYSTEM_TYPE.RAID,
+                block_device=factory.make_PhysicalBlockDevice(node=node))
+            for _ in range(3)
+        ]
+        for _ in range(random.randint(1, 5)):
+            filesystems.append(
+                factory.make_Filesystem(
+                    fstype=FILESYSTEM_TYPE.RAID_SPARE,
+                    block_device=factory.make_PhysicalBlockDevice(node=node)))
+        # Test is that this does not raise an exception.
+        factory.make_FilesystemGroup(
+            group_type=FILESYSTEM_GROUP_TYPE.RAID_10,
+            filesystems=filesystems)
+
+    def test_can_save_raid_10_with_4_or_more_raid_devices_and_spares(self):
+        node = factory.make_Node()
+        filesystems = [
+            factory.make_Filesystem(
+                fstype=FILESYSTEM_TYPE.RAID,
+                block_device=factory.make_PhysicalBlockDevice(node=node))
+            for _ in range(random.randint(4, 10))
+        ]
+        for _ in range(random.randint(1, 5)):
+            filesystems.append(
+                factory.make_Filesystem(
+                    fstype=FILESYSTEM_TYPE.RAID_SPARE,
+                    block_device=factory.make_PhysicalBlockDevice(node=node)))
+        # Test is that this does not raise an exception.
+        factory.make_FilesystemGroup(
+            group_type=FILESYSTEM_GROUP_TYPE.RAID_10,
+            filesystems=filesystems)
+
     def test_cannot_save_bcache_without_cache_set(self):
         node = factory.make_Node()
         filesystems = [
@@ -1319,6 +1402,10 @@
         (FILESYSTEM_GROUP_TYPE.RAID_6, {
             "group_type": FILESYSTEM_GROUP_TYPE.RAID_6,
             "name": "RAID",
+        }),
+        (FILESYSTEM_GROUP_TYPE.RAID_10, {
+            "group_type": FILESYSTEM_GROUP_TYPE.RAID_10,
+            "name": "RAID",
             }),
         (FILESYSTEM_GROUP_TYPE.BCACHE, {
             "group_type": FILESYSTEM_GROUP_TYPE.BCACHE,
@@ -1355,6 +1442,10 @@
         (FILESYSTEM_GROUP_TYPE.RAID_6, {
             "group_type": FILESYSTEM_GROUP_TYPE.RAID_6,
             "prefix": "md",
+        }),
+        (FILESYSTEM_GROUP_TYPE.RAID_10, {
+            "group_type": FILESYSTEM_GROUP_TYPE.RAID_10,
+            "prefix": "md",
             }),
         (FILESYSTEM_GROUP_TYPE.BCACHE, {
             "group_type": FILESYSTEM_GROUP_TYPE.BCACHE,
@@ -1391,6 +1482,10 @@
         (FILESYSTEM_GROUP_TYPE.RAID_6, {
             "group_type": FILESYSTEM_GROUP_TYPE.RAID_6,
             "block_size": 512,
+        }),
+        (FILESYSTEM_GROUP_TYPE.RAID_10, {
+            "group_type": FILESYSTEM_GROUP_TYPE.RAID_10,
+            "block_size": 512,
             }),
         # For BCACHE see
         # `test_get_virtual_block_device_block_size_returns_backing_for_bc`
@@ -1695,7 +1790,7 @@
         block_devices = [
             factory.make_PhysicalBlockDevice(node=node)
             for _ in range(3)
-        ]
+            ]
         uuid = unicode(uuid4())
         with ExpectedException(
                 ValidationError,
@@ -1711,6 +1806,27 @@
                 spare_devices=[],
                 spare_partitions=[])
 
+    def test_create_raid_10_with_2_elements_fails(self):
+        node = factory.make_Node()
+        block_devices = [
+            factory.make_PhysicalBlockDevice(node=node)
+            for _ in range(2)
+        ]
+        uuid = unicode(uuid4())
+        with ExpectedException(
+                ValidationError,
+                re.escape(
+                    "{'__all__': [u'RAID level 10 must have at least 3 raid "
+                    "devices and any number of spares.']}")):
+            RAID.objects.create_raid(
+                name='md0',
+                level=FILESYSTEM_GROUP_TYPE.RAID_10,
+                uuid=uuid,
+                block_devices=block_devices,
+                partitions=[],
+                spare_devices=[],
+                spare_partitions=[])
+
     def test_create_raid_with_block_device_from_other_node_fails(self):
         node1 = factory.make_Node()
         node2 = factory.make_Node()
 
=== modified file 'src/maasserver/preseed_storage.py'
--- src/maasserver/preseed_storage.py	2015-10-27 23:20:28 +0000
+++ src/maasserver/preseed_storage.py	2015-10-29 13:27:52 +0000
@@ -346,6 +346,7 @@
             FILESYSTEM_GROUP_TYPE.RAID_1: 1,
             FILESYSTEM_GROUP_TYPE.RAID_5: 5,
             FILESYSTEM_GROUP_TYPE.RAID_6: 6,
+            FILESYSTEM_GROUP_TYPE.RAID_10: 10,
         }
         return raid_levels[filesystem_group.group_type]
 
 
=== modified file 'src/maasserver/static/js/angular/controllers/node_details_storage.js'
--- src/maasserver/static/js/angular/controllers/node_details_storage.js	2015-10-23 16:34:24 +0000
+++ src/maasserver/static/js/angular/controllers/node_details_storage.js	2015-10-29 13:27:52 +0000
@@ -102,6 +102,15 @@
                 calculateSize: function(minSize, numDisks) {
                     return minSize * (numDisks - 2);
                 }
+            },
+            {
+                level: "raid-10",
+                title: "RAID 10",
+                min_disks: 3,
+                allows_spares: true,
+                calculateSize: function(minSize, numDisks) {
+                    return minSize * numDisks / 2;
+                }
             }
         ];
 
 
=== modified file 'src/maasserver/static/js/angular/controllers/tests/test_node_details_storage.js'
--- src/maasserver/static/js/angular/controllers/tests/test_node_details_storage.js	2015-10-23 18:39:01 +0000
+++ src/maasserver/static/js/angular/controllers/tests/test_node_details_storage.js	2015-10-29 13:27:52 +0000
@@ -3024,9 +3024,10 @@
             var modes = $scope.getAvailableRAIDModes();
             expect(modes[0].level).toEqual("raid-0");
             expect(modes[1].level).toEqual("raid-1");
+            expect(modes.length).toEqual(2);
         });
 
-        it("returns raid 0,1,5 for 3 disks", function() {
+        it("returns raid 0,1,5,10 for 3 disks", function() {
             var controller = makeController();
             $scope.availableNew.devices = [{}, {}, {}];
 
@@ -3034,9 +3035,11 @@
             expect(modes[0].level).toEqual("raid-0");
             expect(modes[1].level).toEqual("raid-1");
             expect(modes[2].level).toEqual("raid-5");
+            expect(modes[3].level).toEqual("raid-10");
+            expect(modes.length).toEqual(4);
         });
 
-        it("returns raid 0,1,5,6 for 4 disks", function() {
+        it("returns raid 0,1,5,6,10 for 4 disks", function() {
             var controller = makeController();
             $scope.availableNew.devices = [{}, {}, {}, {}];
 
@@ -3045,6 +3048,8 @@
             expect(modes[1].level).toEqual("raid-1");
             expect(modes[2].level).toEqual("raid-5");
             expect(modes[3].level).toEqual("raid-6");
+            expect(modes[4].level).toEqual("raid-10");
+            expect(modes.length).toEqual(5);
         });
     });
 
@@ -3070,6 +3075,11 @@
                 level: "raid-6",
                 min_disks: 4,
                 allows_spares: true
+            },
+            {
+                level: "raid-10",
+                min_disks: 3,
+                allows_spares: true
             }
         ];
 
@@ -3323,6 +3333,7 @@
             $scope.availableNew.mode = $scope.getAvailableRAIDModes()[2];
             $scope.setAsSpareRAIDMember(spare0);
 
+            // The 1MB spare causes us to only use 1MB of each active disk.
             expect($scope.getNewRAIDSize()).toBe("2.0 MB");
         });
 
@@ -3358,8 +3369,40 @@
             $scope.availableNew.mode = $scope.getAvailableRAIDModes()[3];
             $scope.setAsSpareRAIDMember(spare0);
 
+            // The 1MB spare causes us to only use 1MB of each active disk.
             expect($scope.getNewRAIDSize()).toBe("2.0 MB");
         });
+
+        it("gets proper raid-10 size", function() {
+            var controller = makeController();
+            var disk0 = {
+                original: {
+                    available_size: 2 * 1000 * 1000
+                }
+            };
+            var disk1 = {
+                original: {
+                    available_size: 2 * 1000 * 1000
+                }
+            };
+            var disk2 = {
+                original: {
+                    available_size: 2 * 1000 * 1000
+                }
+            };
+            var spare0 = {
+                original: {
+                    available_size: 1000 * 1000
+                }
+            };
+            $scope.availableNew.spares = [];
+            $scope.availableNew.devices = [disk0, disk1, disk2, spare0];
+            $scope.availableNew.mode = $scope.getAvailableRAIDModes()[4];
+            $scope.setAsSpareRAIDMember(spare0);
+
+            // The 1MB spare causes us to only use 1MB of each active disk.
+            expect($scope.getNewRAIDSize()).toBe("1.5 MB");
+        });
     });
 
     describe("createRAIDCanSave", function() {
 
=== modified file 'src/maasserver/testing/factory.py'
--- src/maasserver/testing/factory.py	2015-10-28 22:03:20 +0000
+++ src/maasserver/testing/factory.py	2015-10-29 13:27:52 +0000
@@ -1395,6 +1395,18 @@
                     fstype=FILESYSTEM_TYPE.RAID_SPARE,
                     block_device=spare_block_device)
                 group.filesystems.add(spare_filesystem)
+            elif group_type == FILESYSTEM_GROUP_TYPE.RAID_10:
+                for _ in range(4):
+                    block_device = self.make_PhysicalBlockDevice(node)
+                    filesystem = self.make_Filesystem(
+                        fstype=FILESYSTEM_TYPE.RAID,
+                        block_device=block_device)
+                    group.filesystems.add(filesystem)
+                spare_block_device = self.make_PhysicalBlockDevice(node)
+                spare_filesystem = self.make_Filesystem(
+                    fstype=FILESYSTEM_TYPE.RAID_SPARE,
+                    block_device=spare_block_device)
+                group.filesystems.add(spare_filesystem)
             elif group_type == FILESYSTEM_GROUP_TYPE.BCACHE:
                 backing_block_device = self.make_PhysicalBlockDevice(node)
                 backing_filesystem = self.make_Filesystem(
 
=== modified file 'src/maasserver/tests/test_forms_raid.py'
--- src/maasserver/tests/test_forms_raid.py	2015-10-28 01:59:30 +0000
+++ src/maasserver/tests/test_forms_raid.py	2015-10-29 13:27:52 +0000
@@ -261,6 +261,7 @@
                 FILESYSTEM_GROUP_TYPE.RAID_1,
                 FILESYSTEM_GROUP_TYPE.RAID_5,
                 FILESYSTEM_GROUP_TYPE.RAID_6,
+                FILESYSTEM_GROUP_TYPE.RAID_10,
         ]:
             form = CreateRaidForm(node=node, data={
                 'name': 'md1',