Merge lp:~blake-rouse/maas/rsd-compose-storage into lp:~maas-committers/maas/trunk

[Blake Rouse]

[lp:~blake-rouse/maas/rsd-compose-storage]

5936. By Blake Rouse on 2017-04-10

Merge pod-storage discovery.

[MAAS Lander (maas-lander)]

Transitioned to Git.

lp:maas has now moved from Bzr to Git.
Please propose your branches with Launchpad using Git.

git clone https://git.launchpad.net/maas

Unmerged revisions

5936. By Blake Rouse on 2017-04-10

Merge pod-storage discovery.

5935. By Blake Rouse on 2017-04-07

Improve the logic of generating the JSON payload to allocation.

5934. By Blake Rouse on 2017-04-06

Merge trunk.

5933. By Blake Rouse on 2017-04-06

Use newells branch to set the master to clone from.

5932. By Blake Rouse on 2017-04-06

Merge trunk.

5931. By Blake Rouse on 2017-04-06

Merge rsd scrape storage.

5930. By Blake Rouse on 2017-04-06

Add remote storage to compose.

=== modified file 'src/provisioningserver/drivers/pod/rsd.py'
--- src/provisioningserver/drivers/pod/rsd.py	2017-04-06 19:59:20 +0000
+++ src/provisioningserver/drivers/pod/rsd.py	2017-04-10 13:49:22 +0000
@@ -19,6 +19,7 @@
     SETTING_SCOPE,
 )
 from provisioningserver.drivers.pod import (
+    BlockDeviceType,
     Capabilities,
     DiscoveredMachine,
     DiscoveredMachineBlockDevice,
@@ -31,7 +32,10 @@
 )
 from provisioningserver.logger import get_maas_logger
 from provisioningserver.rpc.exceptions import PodInvalidResources
-from provisioningserver.utils.twisted import asynchronous
+from provisioningserver.utils.twisted import (
+    asynchronous,
+    pause,
+)
 from twisted.internet import reactor
 from twisted.internet._sslverify import (
     ClientTLSOptions,
@@ -229,12 +233,7 @@
                 targets.append(remote_drive['@odata.id'])
         return set(targets)
 
-    @inlineCallbacks
-    def calculate_remote_storage(self, url, headers):
-        logical_drives, target_links = (
-            yield self.scrape_logical_drives_and_targets(url, headers))
-        remote_drives = yield self.scrape_remote_drives(url, headers)
-
+    def calculate_remote_storage(self, remote_drives, logical_drives, targets):
         # Find LVGs and LVs out of all logical drives.
         lvgs = {}
         lvs = {}
@@ -251,7 +250,8 @@
             total = 0
             available = 0
             master_id = 0
-            master = None
+            master_size = 0
+            master_path = None
 
             # Find size of LVG and get LVs for this LVG.
             lvg_capacity = lvg_data['CapacityGiB']
@@ -282,8 +282,9 @@
                     if not (lv_target_links & remote_drives):
                         lvs_capacity_unused += lv_capacity
                         new_master_id = int(lv_info['Id'])
-                        if (master is None or master_id > new_master_id):
-                            master = lv_link
+                        if (master_path is None or master_id > new_master_id):
+                            master_path = b"/" + lv_link
+                            master_size = lv_capacity
                             master_id = new_master_id
 
             total = (
@@ -293,10 +294,34 @@
             remote_storage[lvg] = {
                 'total': total,
                 'available': available,
-                'master': master
+                'master': {
+                    'path': master_path,
+                    'size': master_size
+                }
             }
         return remote_storage
 
+    def calculate_pod_remote_storage(
+            self, remote_drives, logical_drives, targets):
+        """Calculate the total sum of LVG capacities in the pod and retrieve the
+        largest LV for the hints.
+        """
+        remote_storage = self.calculate_remote_storage(
+            remote_drives, logical_drives, targets)
+
+        total_capacity = 0
+        for lvg, rs_data in remote_storage.items():
+            total_capacity += rs_data['total']
+
+        hint_capacity = 0
+        for lv, lv_data in logical_drives.items():
+            if lv_data['Mode'] == "LV":
+                if lv_data['CapacityGiB'] > hint_capacity:
+                    hint_capacity = lv_data['CapacityGiB']
+        hint_capacity *= 1024 ** 3
+
+        return total_capacity, hint_capacity
+
     @inlineCallbacks
     def get_pod_memory_resources(self, url, headers, system):
         """Get all the memory resources for the given system."""
@@ -408,7 +433,7 @@
         return discovered_pod
 
     @inlineCallbacks
-    def get_pod_machine(self, node, url, headers):
+    def get_pod_machine(self, node, url, headers, logical_drives, targets):
         """Get pod composed machine.
 
         If required resources cannot be found, this
@@ -425,44 +450,37 @@
         node_data, _ = yield self.redfish_request(
             b"GET", join(url, node), headers)
         # Get hostname.
-        hostname = node_data.get('Name')
-        if hostname is not None:
-            discovered_machine.hostname = hostname
+        discovered_machine.hostname = node_data['Name']
         # Get power state.
-        power_state = node_data.get('PowerState')
-        if power_state is not None:
-            discovered_machine.power_state = RSD_SYSTEM_POWER_STATE.get(
-                power_state)
+        power_state = node_data['PowerState']
+        discovered_machine.power_state = RSD_SYSTEM_POWER_STATE.get(
+            power_state)
         # Get memories.
-        memories = node_data.get('Links', {}).get('Memory')
+        memories = node_data.get('Links', {}).get('Memory', [])
         for memory in memories:
             memory_data, _ = yield self.redfish_request(
                 b"GET", join(url, memory[
                     '@odata.id'].lstrip('/').encode('utf-8')), headers)
-            mem = memory_data.get('CapacityMiB')
-            if mem is not None:
-                discovered_machine.memory += mem
+            discovered_machine.memory += memory_data['CapacityMiB']
         # Get processors.
-        processors = node_data.get('Links', {}).get('Processors')
+        processors = node_data.get('Links', {}).get('Processors', [])
         for processor in processors:
             processor_data, _ = yield self.redfish_request(
                 b"GET", join(url, processor[
                     '@odata.id'].lstrip('/').encode('utf-8')), headers)
             # Using 'TotalThreads' instead of 'TotalCores'
             # as this is what MAAS finds when commissioning.
-            total_threads = processor_data.get('TotalThreads')
-            if total_threads is not None:
-                discovered_machine.cores += total_threads
+            discovered_machine.cores += processor_data['TotalThreads']
             discovered_machine.cpu_speeds.append(
-                processor_data.get('MaxSpeedMHz'))
+                processor_data['MaxSpeedMHz'])
             # Set architecture to first processor
             # architecture type found.
             if not discovered_machine.architecture:
-                arch = processor_data.get('InstructionSet')
+                arch = processor_data['InstructionSet']
                 discovered_machine.architecture = (
                     RSD_ARCH.get(arch, arch))
         # Get local storages.
-        local_drives = node_data.get('Links', {}).get('LocalDrives')
+        local_drives = node_data.get('Links', {}).get('LocalDrives', [])
         for local_drive in local_drives:
             discovered_machine_block_device = (
                 DiscoveredMachineBlockDevice(
@@ -470,38 +488,61 @@
             drive_data, _ = yield self.redfish_request(
                 b"GET", join(url, local_drive[
                     '@odata.id'].lstrip('/').encode('utf-8')), headers)
-            model = drive_data.get('Model')
-            if model is not None:
-                discovered_machine_block_device.model = model
-            serial_number = drive_data.get('SerialNumber')
-            if serial_number is not None:
-                discovered_machine_block_device.serial = (
-                    serial_number)
-            capacity = drive_data.get('CapacityGiB')
-            if capacity is not None:
-                # GiB to Bytes.
-                discovered_machine_block_device.size = float(
-                    capacity) * 1073741824
-            if drive_data.get('Type') == 'SSD':
+            discovered_machine_block_device.model = drive_data['Model']
+            discovered_machine_block_device.serial = drive_data['SerialNumber']
+            discovered_machine_block_device.size = float(
+                drive_data['CapacityGiB']) * (1024 ** 3)
+            if drive_data['Type'] == 'SSD':
                 discovered_machine_block_device.tags = ['ssd']
             discovered_machine.block_devices.append(
                 discovered_machine_block_device)
+        # Get remote storages.
+        remote_drives = node_data.get('Links', {}).get('RemoteDrives', [])
+        for remote_drive in remote_drives:
+            discovered_machine_block_device = (
+                DiscoveredMachineBlockDevice(
+                    model=None, serial=None, size=0,
+                    type=BlockDeviceType.ISCSI))
+            target_data = targets[
+                remote_drive['@odata.id'].lstrip('/').encode('utf-8')]
+            addresses = target_data.get('Addresses')[0]
+            host = addresses.get('iSCSI', {}).get('TargetPortalIP')
+            proto = '6'  # curtin currently only supports TCP.
+            port = str(addresses.get('iSCSI', {}).get('TargetPortalPort'))
+            luns = addresses.get('iSCSI', {}).get('TargetLUN', [])
+            if luns:
+                lun = str(luns[0]['LUN'])
+            else:
+                # Set LUN to 0 if not available.
+                lun = '0'
+            target_name = addresses.get('iSCSI', {}).get('TargetIQN')
+            discovered_machine_block_device.iscsi_target = ':'.join(
+                [host, proto, port, lun, target_name])
+            discovered_machine_block_device.tags = ['iscsi']
+            # Loop through all the logical drives till we
+            # find which one contains this remote drive.
+            for lv, lv_data in logical_drives.items():
+                lv_targets = lv_data.get('Links', {}).get('Targets', [])
+                if remote_drive in lv_targets:
+                    discovered_machine_block_device.size = float(
+                        lv_data['CapacityGiB']) * (1024 ** 3)
+            discovered_machine.block_devices.append(
+                discovered_machine_block_device)
         # Get interfaces.
-        interfaces = node_data.get('Links', {}).get('EthernetInterfaces')
+        interfaces = node_data.get('Links', {}).get('EthernetInterfaces', [])
         for interface in interfaces:
             discovered_machine_interface = DiscoveredMachineInterface(
                 mac_address='')
             interface_data, _ = yield self.redfish_request(
                 b"GET", join(url, interface[
                     '@odata.id'].lstrip('/').encode('utf-8')), headers)
-            mac_address = interface_data.get('MACAddress')
-            if mac_address is not None:
-                discovered_machine_interface.mac_address = mac_address
-            nic_speed = interface_data.get('SpeedMbps')
+            discovered_machine_interface.mac_address = (
+                interface_data['MACAddress'])
+            nic_speed = interface_data['SpeedMbps']
             if nic_speed is not None:
                 if nic_speed < 1000:
                     discovered_machine_interface.tags = ["e%s" % nic_speed]
-                elif nic_speed == "1000":
+                elif nic_speed == 1000:
                     discovered_machine_interface.tags = ["1g", "e1000"]
                 else:
                     # We know that the Mbps > 1000
@@ -522,9 +563,8 @@
                                 vlan = vlan.lstrip('/').encode('utf-8')
                                 vlan_data, _ = yield self.redfish_request(
                                     b"GET", join(url, vlan), headers)
-                                vlan_id = vlan_data.get('VLANId')
-                                if vlan_id is not None:
-                                    discovered_machine_interface.vid = vlan_id
+                                discovered_machine_interface.vid = (
+                                    vlan_data['VLANId'])
             else:
                 # If no NeighborPort, this interface is on
                 # the management network.
@@ -547,7 +587,7 @@
         return discovered_machine
 
     @inlineCallbacks
-    def get_pod_machines(self, url, headers):
+    def get_pod_machines(self, url, headers, logical_drives, targets):
         """Get pod composed machines.
 
         If required resources cannot be found, these
@@ -560,7 +600,8 @@
         nodes = yield self.list_resources(nodes_uri, headers)
         # Iterate over all composed nodes in the pod.
         for node in nodes:
-            discovered_machine = yield self.get_pod_machine(node, url, headers)
+            discovered_machine = yield self.get_pod_machine(
+                node, url, headers, logical_drives, targets)
             discovered_machines.append(discovered_machine)
         return discovered_machines
 
@@ -573,8 +614,6 @@
             for cpu_speed in machine.cpu_speeds:
                 if cpu_speed in discovered_pod.cpu_speeds:
                     discovered_pod.cpu_speeds.remove(cpu_speed)
-            # Delete cpu_speeds place holder.
-            del machine.cpu_speeds
             used_cores += machine.cores
             used_memory += machine.memory
             for blk_dev in machine.block_devices:
@@ -600,22 +639,50 @@
         """
         url = self.get_url(context)
         headers = self.make_auth_headers(**context)
+        logical_drives, targets = yield self.scrape_logical_drives_and_targets(
+            url, headers)
+        remote_drives = yield self.scrape_remote_drives(url, headers)
+        pod_remote_storage, pod_hints_remote_storage = (
+            self.calculate_pod_remote_storage(
+                remote_drives, logical_drives, targets))
 
         # Discover pod resources.
         discovered_pod = yield self.get_pod_resources(url, headers)
 
+        # Discover pod remote storage resources.
+        discovered_pod.capabilities.append(Capabilities.ISCSI_STORAGE)
+        discovered_pod.iscsi_storage = pod_remote_storage
+
         # Discover composed machines.
         discovered_pod.machines = yield self.get_pod_machines(
             url, headers)
 
         # Discover pod hints.
         discovered_pod.hints = self.get_pod_hints(discovered_pod)
+        discovered_pod.hints.iscsi_storage = pod_hints_remote_storage
 
-        # Delete cpu_speeds place holder.
-        del discovered_pod.cpu_speeds
         return discovered_pod
 
-    def convert_request_to_json_payload(self, processors, cores, request):
+    def select_remote_master(self, remote_storage, size):
+        """Select the remote master drive that has enough space."""
+        for lvg, data in remote_storage.items():
+            if data['master'] and data['available'] >= size:
+                data['available'] -= size
+                return data['master']
+
+    def set_drive_type(self, drive, block_device):
+        """Set type of drive requested on `drive` based on the tags on
+        `block_device`."""
+        if 'ssd' in block_device.tags:
+            drive['Type'] = 'SSD'
+        elif 'nvme' in block_device.tags:
+            drive['Type'] = 'NVMe'
+        elif 'hdd' in block_device.tags:
+            drive['Type'] = 'HDD'
+
+    def convert_request_to_json_payload(
+            self, processors, cores, request,
+            remote_drives, logical_drives, targets):
         """Convert the RequestedMachine object to JSON."""
         # The below fields are for RSD allocation.
         # Most of these fields are nullable and could be used at
@@ -643,6 +710,14 @@
             "SerialNumber": None,
             "Interface": None,
         }
+        remote_drive = {
+            "CapacityGiB": None,
+            "iSCSIAddress": None,
+            "Master": {
+                "Type": "Clone",
+                "Resource": None,
+            },
+        }
         interface = {
             "SpeedMbps": None,
             "PrimaryVLAN": None,
@@ -652,41 +727,108 @@
             "Processors": [],
             "Memory": [],
             "LocalDrives": [],
+            "RemoteDrives": [],
             "EthernetInterfaces": [],
         }
-        request = request.asdict()
 
         # Processors.
         for _ in range(processors):
             proc = processor.copy()
             proc['TotalCores'] = cores
-            arch = request.get('architecture')
+            arch = request.architecture
             for key, val in RSD_ARCH.items():
                 if val == arch:
                     proc['InstructionSet'] = key
             # cpu_speed is only optional field in request.
-            cpu_speed = request.get('cpu_speed')
+            cpu_speed = request.cpu_speed
             if cpu_speed is not None:
                 proc['AchievableSpeedMHz'] = cpu_speed
             data['Processors'].append(proc)
 
+        # Determine remote storage information if more than one driver is
+        # requested.
+        remote_storage = None
+        if len(request.block_devices) > 1:
+            remote_storage = self.calculate_remote_storage(
+                remote_drives, logical_drives, targets)
+
         # Block Devices.
-        block_devices = request.get('block_devices')
-        for block_device in block_devices:
-            drive = local_drive.copy()
-            # Convert from bytes to GiB.
-            drive['CapacityGiB'] = block_device['size'] / 1073741824
-            data['LocalDrives'].append(drive)
+        #
+        # Tags are matched on the block devices to create different types
+        # of requested storage for a block device.
+        #     local: Locally attached disk (aka. LocalDrive).
+        #     ssd: Locally attached SSD (aka. LocalDrive).
+        #     hdd: Locally attached HDD (aka. LocalDrive).
+        #     nvme: Locally attached NVMe (aka. LocalDrive).
+        #     iscsi: Remotely attached disk over ISCSI (aka. RemoteTarget)
+        #     (none): Remotely attached disk will be picked unless its the
+        #             first disk. First disk is locally attached.
+        block_devices = request.block_devices
+        boot_disk = True
+        for idx, block_device in enumerate(block_devices):
+            if boot_disk:
+                if 'iscsi' in block_device.tags:
+                    raise PodActionError(
+                        'iSCSI is not supported as being a boot disk.')
+                else:
+                    # Force 'local' into the tags if not present.
+                    if 'local' not in block_device.tags:
+                        block_device.tags.append('local')
+                    drive = local_drive.copy()
+                    # Convert from bytes to GiB.
+                    drive['CapacityGiB'] = block_device.size / (1024 ** 3)
+                    self.set_drive_type(drive, block_device.tags)
+                    data['LocalDrives'].append(drive)
+                boot_disk = False
+            else:
+                is_local = max(
+                    tag in block_device.tags
+                    for tag in ['local', 'ssd', 'nvme', 'hdd']
+                )
+                if is_local:
+                    # Force the local tag if it wasn't provided.
+                    if 'local' not in block_device.tags:
+                        block_device.tags.append('local')
+                    drive = local_drive.copy()
+                    # Convert from bytes to GiB.
+                    drive['CapacityGiB'] = block_device.size / (1024 ** 3)
+                    self.set_drive_type(drive, block_device.tags)
+                    data['LocalDrives'].append(drive)
+                else:
+                    # Force 'iscsi' into the tags if not present.
+                    if 'iscsi' not in block_device.tags:
+                        block_device.tags.append('iscsi')
+                    size = block_device.size / (1024 ** 3)
+                    # Determine the remote master that can be used.
+                    remote_master = self.select_remote_master(
+                        remote_storage, size)
+                    if remote_master is None:
+                        raise PodActionError(
+                            'iSCSI remote drive cannot be created because '
+                            'not enough space is available.')
+                    drive = remote_drive.copy()
+                    # Convert from bytes to GiB.
+                    drive['CapacityGiB'] = size
+                    drive['iSCSIAddress'] = 'iqn.2010-08.io.maas:%s-%s' % (
+                        request.hostname, idx)
+                    drive['Master']['Resource'] = {
+                        '@odata.id': remote_master.decode('utf-8'),
+                    }
+                    data['RemoteDrives'].append(drive)
+                    # Save the iSCSIAddress on the RequestBlockDevice. This is
+                    # used to map the DiscoveredMachineBlockDevice tags to
+                    # the same tags used during the request.
+                    block_device.iscsi_target = drive['iSCSIAddress']
 
         # Interfaces.
-        interfaces = request.get('interfaces')
+        interfaces = request.interfaces
         for iface in interfaces:
             nic = interface.copy()
             data['EthernetInterfaces'].append(nic)
 
         # Memory.
         mem = memory.copy()
-        mem['CapacityMiB'] = request.get('memory')
+        mem['CapacityMiB'] = request.memory
         data['Memory'].append(mem)
 
         return json.dumps(data).encode('utf-8')
@@ -697,6 +839,9 @@
         url = self.get_url(context)
         headers = self.make_auth_headers(**context)
         endpoint = b"redfish/v1/Nodes/Actions/Allocate"
+        logical_drives, targets = (
+            yield self.scrape_logical_drives_and_targets(url, headers))
+        remote_drives = yield self.scrape_remote_drives(url, headers)
         # Create allocate payload.
         requested_cores = request.cores
         if requested_cores % 2 != 0:
@@ -709,9 +854,11 @@
         # Find the correct procesors and cores combination from RSD POD.
         processors = 1
         cores = requested_cores
+        response_headers = None
         while True:
             payload = self.convert_request_to_json_payload(
-                processors, cores, request)
+                processors, cores, request, remote_drives,
+                logical_drives, targets)
             try:
                 _, response_headers = yield self.redfish_request(
                     b"POST", join(url, endpoint), headers,
@@ -732,14 +879,16 @@
             node_id = location[0].rsplit('/', 1)[-1]
             node_path = location[0].split('/', 3)[-1]
 
-            # Retrieve new node.
-            discovered_machine = yield self.get_pod_machine(
-                node_path.encode('utf-8'), url, headers)
             # Assemble the node.
             yield self.assemble_node(url, node_id.encode('utf-8'), headers)
             # Set to PXE boot.
             yield self.set_pxe_boot(url, node_id.encode('utf-8'), headers)
 
+            # Retrieve new node.
+            discovered_machine = yield self.get_pod_machine(
+                node_path.encode('utf-8'), url,
+                headers, logical_drives, targets)
+
             # Retrieve pod resources.
             discovered_pod = yield self.get_pod_resources(url, headers)
             # Retrive pod hints.
@@ -830,8 +979,11 @@
         node_state = yield self.get_composed_node_state(
             url, node_id, headers)
         while node_state == 'Assembling':
+            # Wait 2 seconds before getting updated state.
+            yield pause(2)
             node_state = yield self.get_composed_node_state(
                 url, node_id, headers)
+
         # Check one last time if the state has became `Failed`.
         if node_state == 'Failed':
             # Broken system.
 
=== modified file 'src/provisioningserver/drivers/pod/tests/test_rsd.py'
--- src/provisioningserver/drivers/pod/tests/test_rsd.py	2017-04-06 19:26:13 +0000
+++ src/provisioningserver/drivers/pod/tests/test_rsd.py	2017-04-10 13:49:22 +0000
@@ -28,6 +28,7 @@
     MAASTwistedRunTest,
 )
 from provisioningserver.drivers.pod import (
+    BlockDeviceType,
     Capabilities,
     DiscoveredMachine,
     DiscoveredMachineBlockDevice,
@@ -50,6 +51,7 @@
 from testtools import ExpectedException
 from testtools.matchers import (
     Equals,
+    Is,
     MatchesDict,
     MatchesListwise,
     MatchesStructure,
@@ -279,6 +281,8 @@
         }],
         "RemoteDrives": [{
             "@odata.id": "/redfish/v1/Services/1/Targets/1"
+        }, {
+            "@odata.id": "/redfish/v1/Services/1/Targets/2"
         }],
         "ManagedBy": [{
             "@odata.id": "/redfish/v1/Managers/1"
@@ -600,6 +604,8 @@
         }],
         "Targets": [{
             "@odata.id": "/redfish/v1/Services/1/Targets/1"
+        }, {
+            "@odata.id": "/redfish/v1/Services/1/Targets/2"
         }]
     }
 }
@@ -958,14 +964,12 @@
         mock_redfish_request.return_value = (SAMPLE_JSON_NODE, None)
 
         remote_drives = yield driver.scrape_remote_drives(url, headers)
-        self.assertEquals({'/redfish/v1/Services/1/Targets/1'}, remote_drives)
+        self.assertEquals({
+            '/redfish/v1/Services/1/Targets/1',
+            '/redfish/v1/Services/1/Targets/2'}, remote_drives)
 
-    @inlineCallbacks
     def test__calculate_remote_storage(self):
         driver = RSDPodDriver()
-        context = make_context()
-        url = driver.get_url(context)
-        headers = driver.make_auth_headers(**context)
         LV_NO_TARGETS = deepcopy(SAMPLE_JSON_LV)
         LV_NO_TARGETS['Links']['Targets'] = []
         logical_drives = {
@@ -975,31 +979,64 @@
             b"redfish/v1/Services/1/LogicalDrives/4": SAMPLE_JSON_PV,
             b"redfish/v1/Services/1/LogicalDrives/5": SAMPLE_JSON_PV,
         }
-        target_links = {
+        targets = {
             b"redfish/v1/Services/1/Targets/1": SAMPLE_JSON_TARGET,
             b"redfish/v1/Services/1/Targets/2": SAMPLE_JSON_TARGET,
             b"redfish/v1/Services/1/Targets/3": SAMPLE_JSON_TARGET,
         }
         remote_drives = set(
             "redfish/v1/Services/1/Targets/1")
-        mock_scrape_logical_drives_and_targets = self.patch(
-            driver, 'scrape_logical_drives_and_targets')
-        mock_scrape_logical_drives_and_targets.return_value = (
-            logical_drives, target_links)
-        mock_scrape_remote_drives = self.patch(driver, 'scrape_remote_drives')
-        mock_scrape_remote_drives.return_value = remote_drives
 
-        remote_storage = yield driver.calculate_remote_storage(url, headers)
+        remote_storage = driver.calculate_remote_storage(
+            remote_drives, logical_drives, targets)
         self.assertDictEqual(
             remote_storage,
             {
                 b'redfish/v1/Services/1/LogicalDrives/2': {
                     'total': 11830638411776.0,
                     'available': 11830638411776.0,
-                    'master': b'redfish/v1/Services/1/LogicalDrives/1'
+                    'master': {
+                        'path': b'/redfish/v1/Services/1/LogicalDrives/1',
+                        'size': 80
+                    }
                 }
             })
 
+    def test__calculate_pod_remote_storage(self):
+        driver = RSDPodDriver()
+        logical_drives = {
+            b"redfish/v1/Services/1/LogicalDrives/1": SAMPLE_JSON_LV,
+            b"redfish/v1/Services/1/LogicalDrives/2": SAMPLE_JSON_LVG,
+            b"redfish/v1/Services/1/LogicalDrives/3": SAMPLE_JSON_LV
+        }
+        remote_storage = {
+            b"redfish/v1/Services/1/LogicalDrives/1": {
+                'total': 80 * (1024 ** 3),
+                'available': None,
+                'master': {
+                    'path': None,
+                    'size': None
+                }
+            },
+            b"redfish/v1/Services/1/LogicalDrives/2": {
+                'total': 12 * (1024 ** 3),
+                'available': None,
+                'master': {
+                    'path': None,
+                    'size': None
+                }
+            }
+        }
+        mock_calculate_remote_storage = self.patch(
+            driver, 'calculate_remote_storage')
+        mock_calculate_remote_storage.return_value = remote_storage
+
+        pod_capacity, pod_hints_capacity = driver.calculate_pod_remote_storage(
+            factory.make_name('remote_drives'),
+            logical_drives, factory.make_name('targets'))
+        self.assertEquals(92 * (1024 ** 3), pod_capacity)
+        self.assertEquals(80 * (1024 ** 3), pod_hints_capacity)
+
     @inlineCallbacks
     def test__get_pod_memory_resources(self):
         driver = RSDPodDriver()
@@ -1147,10 +1184,6 @@
         url = driver.get_url(context)
         headers = driver.make_auth_headers(**context)
         mock_redfish_request = self.patch(driver, 'redfish_request')
-        NO_MEMORY = deepcopy(SAMPLE_JSON_MEMORY)
-        NO_MEMORY['CapacityMiB'] = None
-        NO_THREADS = deepcopy(SAMPLE_JSON_PROCESSOR)
-        NO_THREADS['TotalThreads'] = None
         NIC1_DATA = deepcopy(SAMPLE_JSON_INTERFACE)
         NIC1_DATA['SpeedMbps'] = 900
         NIC2_DATA = deepcopy(SAMPLE_JSON_INTERFACE)
@@ -1162,10 +1195,10 @@
         mock_redfish_request.side_effect = [
             (SAMPLE_JSON_NODE, None),
             (SAMPLE_JSON_MEMORY, None),
-            (NO_MEMORY, None),
-            (SAMPLE_JSON_MEMORY, None),
-            (SAMPLE_JSON_MEMORY, None),
-            (NO_THREADS, None),
+            (SAMPLE_JSON_MEMORY, None),
+            (SAMPLE_JSON_MEMORY, None),
+            (SAMPLE_JSON_MEMORY, None),
+            (SAMPLE_JSON_PROCESSOR, None),
             (SAMPLE_JSON_PROCESSOR, None),
             (SAMPLE_JSON_DEVICE, None),
             (SAMPLE_JSON_DEVICE, None),
@@ -1182,46 +1215,98 @@
             (SAMPLE_JSON_PORT, None),
             (SAMPLE_JSON_VLAN, None),
         ]
+        LV_NO_TARGETS = deepcopy(SAMPLE_JSON_LV)
+        LV_NO_TARGETS['Links']['Targets'] = []
+        TARGET_LUN = deepcopy(SAMPLE_JSON_TARGET)
+        TARGET_LUN['Addresses'][0]['iSCSI']['TargetLUN'].append({'LUN': 3})
+        logical_drives = {
+            b"redfish/v1/Services/1/LogicalDrives/1": SAMPLE_JSON_LV,
+            b"redfish/v1/Services/1/LogicalDrives/2": SAMPLE_JSON_LVG,
+            b"redfish/v1/Services/1/LogicalDrives/3": LV_NO_TARGETS,
+            b"redfish/v1/Services/1/LogicalDrives/4": SAMPLE_JSON_PV,
+            b"redfish/v1/Services/1/LogicalDrives/5": SAMPLE_JSON_PV,
+        }
+        targets = {
+            b"redfish/v1/Services/1/Targets/1": SAMPLE_JSON_TARGET,
+            b"redfish/v1/Services/1/Targets/2": TARGET_LUN,
+            b"redfish/v1/Services/1/Targets/3": SAMPLE_JSON_TARGET,
+            b"redfish/v1/Services/1/Targets/4": SAMPLE_JSON_TARGET,
+        }
 
         machine = yield driver.get_pod_machine(
-            b"redfish/v1/Nodes/1", url, headers)
-        self.assertEquals("amd64/generic", machine.architecture)
-        self.assertEquals(28, machine.cores)
-        self.assertEquals(2300, machine.cpu_speed)
-        self.assertEquals(23436, machine.memory)
-        self.assertEquals(
-            "INTEL_SSDMCEAC120B3", machine.block_devices[0].model)
-        self.assertEquals("CVLI310601PY120E", machine.block_devices[0].serial)
-        self.assertEquals(119999999999.99997, machine.block_devices[0].size)
-        self.assertEquals(['ssd'], machine.block_devices[0].tags)
-        self.assertEquals("off", machine.power_state)
-        self.assertEquals({'node_id': '1'}, machine.power_parameters)
-        self.assertThat(machine.interfaces, MatchesListwise([
-            MatchesStructure(
-                mac_address=Equals('54:ab:3a:36:af:45'),
-                vid=Equals(4088),
-                tags=Equals(['e900']),
-                boot=Equals(False),
-            ),
-            MatchesStructure(
-                mac_address=Equals('54:ab:3a:36:af:45'),
-                vid=Equals(4088),
-                tags=Equals(['1.0']),
-                boot=Equals(False),
-            ),
-            MatchesStructure(
-                mac_address=Equals('54:ab:3a:36:af:45'),
-                vid=Equals(4088),
-                tags=Equals(['2.0']),
-                boot=Equals(False),
-            ),
-            MatchesStructure(
-                mac_address=Equals('54:ab:3a:36:af:45'),
-                vid=Equals(-1),
-                tags=Equals([]),
-                boot=Equals(True),
-            ),
-        ]))
+            b"redfish/v1/Nodes/1", url, headers, logical_drives, targets)
+        self.assertThat(machine, MatchesStructure(
+            architecture=Equals("amd64/generic"),
+            cores=Equals(56),
+            cpu_speed=Equals(2300),
+            memory=Equals(31248),
+            power_state=Equals("off"),
+            power_parameters=MatchesDict({'node_id': Equals('1')}),
+            interfaces=MatchesListwise([
+                MatchesStructure(
+                    mac_address=Equals('54:ab:3a:36:af:45'),
+                    vid=Equals(4088),
+                    tags=Equals(['e900']),
+                    boot=Equals(False),
+                ),
+                MatchesStructure(
+                    mac_address=Equals('54:ab:3a:36:af:45'),
+                    vid=Equals(4088),
+                    tags=Equals(['1g', 'e1000']),
+                    boot=Equals(False),
+                ),
+                MatchesStructure(
+                    mac_address=Equals('54:ab:3a:36:af:45'),
+                    vid=Equals(4088),
+                    tags=Equals(['2.0']),
+                    boot=Equals(False),
+                ),
+                MatchesStructure(
+                    mac_address=Equals('54:ab:3a:36:af:45'),
+                    vid=Equals(-1),
+                    tags=Equals([]),
+                    boot=Equals(True),
+                ),
+            ]),
+            block_devices=MatchesListwise([
+                MatchesStructure(
+                    model=Equals('INTEL_SSDMCEAC120B3'),
+                    serial=Equals('CVLI310601PY120E'),
+                    size=Equals(119999999999.99997),
+                    block_size=Equals(512),
+                    tags=Equals(['ssd']),
+                    type=Equals(BlockDeviceType.PHYSICAL),
+                ),
+                MatchesStructure(
+                    model=Equals('INTEL_SSDMCEAC120B3'),
+                    serial=Equals('CVLI310601PY120E'),
+                    size=Equals(119999999999.99997),
+                    block_size=Equals(512),
+                    tags=Equals(['ssd']),
+                    type=Equals(BlockDeviceType.PHYSICAL),
+                ),
+                MatchesStructure(
+                    model=Is(None),
+                    serial=Is(None),
+                    size=Equals(85899345920.0),
+                    block_size=Equals(512),
+                    tags=Equals(['iscsi']),
+                    type=Equals(BlockDeviceType.ISCSI),
+                    iscsi_target=Equals(
+                        '10.1.0.100:6:3260:0:iqn.maas.io:test'),
+                ),
+                MatchesStructure(
+                    model=Is(None),
+                    serial=Is(None),
+                    size=Equals(85899345920.0),
+                    block_size=Equals(512),
+                    tags=Equals(['iscsi']),
+                    type=Equals(BlockDeviceType.ISCSI),
+                    iscsi_target=Equals(
+                        '10.1.0.100:6:3260:3:iqn.maas.io:test'),
+                )
+            ])
+        ))
 
     @inlineCallbacks
     def test__get_pod_machines(self):
@@ -1229,6 +1314,14 @@
         context = make_context()
         url = driver.get_url(context)
         headers = driver.make_auth_headers(**context)
+        logical_drives = {
+            factory.make_name('lv_path'): factory.make_name('lv_data')
+            for _ in range(3)
+        }
+        targets = {
+            factory.make_name('target_path'): factory.make_name('target_data')
+            for _ in range(3)
+        }
         mock_list_resources = self.patch(driver, 'list_resources')
         mock_list_resources.side_effect = [
             [b"redfish/v1/Nodes/1"],
@@ -1242,10 +1335,11 @@
         mock_get_pod_machine = self.patch(driver, 'get_pod_machine')
         mock_get_pod_machine.return_value = expected_machines
 
-        discovered_machines = yield driver.get_pod_machines(url, headers)
+        discovered_machines = yield driver.get_pod_machines(
+            url, headers, logical_drives, targets)
         self.assertEquals(1, len(discovered_machines))
         self.assertThat(mock_get_pod_machine, MockCalledOnceWith(
-            b"redfish/v1/Nodes/1", url, headers))
+            b"redfish/v1/Nodes/1", url, headers, logical_drives, targets))
 
     def test__get_pod_hints(self):
         driver = RSDPodDriver()
@@ -1283,12 +1377,41 @@
         context = make_context()
         headers = driver.make_auth_headers(**context)
         url = driver.get_url(context)
+        remote_drives = factory.make_name('remote_drive')
+        logical_drives = factory.make_name('logical_drives')
+        targets = factory.make_name('targets')
+        pod_iscsi_capacity = random.randint(
+            10 * 1024 ** 3, 20 * 1024 ** 3)
+        pod_hints_iscsi_capacity = random.randint(
+            10 * 1024 ** 3, 20 * 1024 ** 3)
+        mock_scrape_logical_drives_and_targets = self.patch(
+            driver, 'scrape_logical_drives_and_targets')
+        mock_scrape_logical_drives_and_targets.return_value = (
+            logical_drives, targets)
+        mock_scrape_remote_drives = self.patch(driver, 'scrape_remote_drives')
+        mock_scrape_remote_drives.return_value = remote_drives
+        mock_calculate_pod_remote_storage = self.patch(
+            driver, 'calculate_pod_remote_storage')
+        mock_calculate_pod_remote_storage.return_value = (
+            pod_iscsi_capacity, pod_hints_iscsi_capacity)
         mock_get_pod_resources = self.patch(
             driver, 'get_pod_resources')
         mock_get_pod_machines = self.patch(driver, 'get_pod_machines')
         mock_get_pod_hints = self.patch(driver, 'get_pod_hints')
 
-        yield driver.discover(factory.make_name('system_id'), context)
+        discovered_pod = yield driver.discover(
+            factory.make_name('system_id'), context)
+        self.assertEquals(pod_iscsi_capacity, discovered_pod.iscsi_storage)
+        self.assertEquals(
+            pod_hints_iscsi_capacity, discovered_pod.hints.iscsi_storage)
+        self.assertThat(
+            mock_scrape_logical_drives_and_targets, MockCalledOnceWith(
+                url, headers))
+        self.assertThat(
+            mock_scrape_remote_drives, MockCalledOnceWith(url, headers))
+        self.assertThat(
+            mock_calculate_pod_remote_storage, MockCalledOnceWith(
+                remote_drives, logical_drives, targets))
         self.assertThat(
             mock_get_pod_resources, MockCalledOnceWith(url, headers))
         self.assertThat(
@@ -1302,7 +1425,7 @@
         processors = 2
         cores = request.cores / 2
         payload = driver.convert_request_to_json_payload(
-            processors, cores, request)
+            processors, cores, request, None, None, None)
         self.assertThat(
             json.loads(payload.decode('utf-8')),
             MatchesDict({
@@ -1383,6 +1506,25 @@
         request = make_requested_machine(cores=64)
         discovered_pod = make_discovered_pod()
         new_machine = make_discovered_machine()
+        logical_drives = {
+            factory.make_name('lv_path'): factory.make_name('lv_data')
+            for _ in range(3)
+        }
+        targets = {
+            factory.make_name('target_path'): factory.make_name('target_data')
+            for _ in range(3)
+        }
+        remote_drives = set([
+            factory.make_name('target_path')
+            for _ in range(3)
+        ])
+        mock_scrape_logical_drives_and_targets = self.patch(
+            driver, 'scrape_logical_drives_and_targets')
+        mock_scrape_logical_drives_and_targets.return_value = (
+            logical_drives, targets)
+        mock_scrape_remote_drives = self.patch(
+            driver, 'scrape_remote_drives')
+        mock_scrape_remote_drives.return_value = remote_drives
         mock_get_pod_machine = self.patch(driver, 'get_pod_machine')
         mock_get_pod_machine.return_value = new_machine
         mock_convert_request_to_json_payload = self.patch(
@@ -1408,8 +1550,10 @@
             factory.make_name('system_id'), context, request)
         self.assertThat(
             mock_convert_request_to_json_payload, MockCallsMatch(
-                call(1, 32, request), call(2, 16, request),
-                call(4, 8, request), call(8, 4, request)))
+                call(1, 32, request, remote_drives, logical_drives, targets),
+                call(2, 16, request, remote_drives, logical_drives, targets),
+                call(4, 8, request, remote_drives, logical_drives, targets),
+                call(8, 4, request, remote_drives, logical_drives, targets)))
         self.assertThat(mock_assemble_node, MockCalledOnceWith(
             url, new_machine.power_parameters.get(
                 'node_id').encode('utf-8'), headers))
@@ -1427,6 +1571,12 @@
         discovered_pod = make_discovered_pod()
         new_machines = deepcopy(discovered_pod.machines)
         machines = deepcopy(new_machines)
+        mock_scrape_logical_drives_and_targets = self.patch(
+            driver, 'scrape_logical_drives_and_targets')
+        mock_scrape_logical_drives_and_targets.return_value = (None, None)
+        mock_scrape_remote_drives = self.patch(
+            driver, 'scrape_remote_drives')
+        mock_scrape_remote_drives.return_value = None
         mock_get_pod_machines = self.patch(driver, 'get_pod_machines')
         mock_get_pod_machines.side_effect = [
             machines, new_machines]