Merge lp:~blake-rouse/maas/fix-1378527-1.7 into lp:maas/1.7

[Blake Rouse]

Fixes the images page to not show multiple resources for HWE enablement resources. Fixes the incorrect progress of downloading resources.

[Christian Reis (kiko)]

Ugh, this is a big change.

[Raphaël Badin (rvb)]

Not done with the review yet but here is a first set of comments…

[Raphaël Badin (rvb)]

review: Approve

[Christian Reis (kiko)]

Can someone independently verify trunk before we land this and/or cut the RC for Utopic?

[Andres Rodriguez (andreserl)]

Tested... no regressions. Seems to fix the issues.

review: Approve

[Gavin Panella (allenap)]

I've skimmed over this, so no vote. I don't think I can add anything to what Raphaël has written.

review: Abstain

[Christian Reis (kiko)]

How can this have not regressed locally and then died when on the branch?

review: Needs Information

=== modified file 'src/maasserver/models/bootresource.py'
--- src/maasserver/models/bootresource.py	2014-10-10 10:24:21 +0000
+++ src/maasserver/models/bootresource.py	2014-10-17 01:59:55 +0000
@@ -186,19 +186,28 @@
             else:
                 rtypes = [BOOT_RESOURCE_TYPE.UPLOADED]
                 name = image['release']
-            resource = resources.filter(
+            matching_resources = resources.filter(
                 rtype__in=rtypes, name=name,
-                architecture__startswith=image['architecture']).first()
-            if resource is None:
-                continue
-            if not resource.supports_subarch(image['subarchitecture']):
-                continue
-            resource_set = resource.get_latest_complete_set()
-            if resource_set is None:
-                continue
-            if resource_set.label != image['label']:
-                continue
-            matched_resources.add(resource)
+                architecture__startswith=image['architecture'])
+            for resource in matching_resources:
+                if resource is None:
+                    # This shouldn't happen at all, but just to be sure.
+                    continue
+                if not resource.supports_subarch(image['subarchitecture']):
+                    # This matching resource doesn't support the images
+                    # subarchitecture, so its not a matching resource.
+                    continue
+                resource_set = resource.get_latest_complete_set()
+                if resource_set is None:
+                    # Possible that the import just started, and there is no
+                    # set. Making it not a matching resource, as it cannot
+                    # exist on the cluster unless it has a set.
+                    continue
+                if resource_set.label != image['label']:
+                    # The label is different so the cluster has a different
+                    # version of this set.
+                    continue
+                matched_resources.add(resource)
         return list(matched_resources)
 
     def boot_images_are_in_sync(self, images):

=== modified file 'src/maasserver/models/bootresourceset.py'
--- src/maasserver/models/bootresourceset.py	2014-08-31 02:47:48 +0000
+++ src/maasserver/models/bootresourceset.py	2014-10-17 01:59:55 +0000
@@ -130,7 +130,7 @@
         if size <= 0:
             # Handle division by zero
             return 0
-        return self.total_size / float(size)
+        return 100.0 * size / float(self.total_size)
 
     @property
     def complete(self):

=== modified file 'src/maasserver/models/tests/test_bootresource.py'
--- src/maasserver/models/tests/test_bootresource.py	2014-10-10 10:59:28 +0000
+++ src/maasserver/models/tests/test_bootresource.py	2014-10-17 01:59:55 +0000
@@ -363,6 +363,34 @@
             [resource],
             BootResource.objects.get_resources_matching_boot_images(images))
 
+    def test__returns_multiple_resource_for_hwe_resources(self):
+        os = factory.make_name('os')
+        series = factory.make_name('series')
+        name = '%s/%s' % (os, series)
+        arch = factory.make_name('arch')
+        subarches = [factory.make_name('hwe') for _ in range(3)]
+        resources = [
+            factory.make_usable_boot_resource(
+                rtype=BOOT_RESOURCE_TYPE.SYNCED,
+                name=name, architecture='%s/%s' % (arch, subarch))
+            for subarch in subarches
+            ]
+        images = []
+        for resource in resources:
+            label = resource.get_latest_complete_set().label
+            purposes = [factory.make_name('purpose') for _ in range(3)]
+            arch, subarch = resource.split_arch()
+            images.extend([
+                make_rpc_boot_image(
+                    osystem=os, release=series,
+                    architecture=arch, subarchitecture=subarch,
+                    label=label, purpose=purpose)
+                for purpose in purposes
+                ])
+        self.assertItemsEqual(
+            resources,
+            BootResource.objects.get_resources_matching_boot_images(images))
+
     def test__returns_resource_for_generated_resource(self):
         resource = factory.make_usable_boot_resource(
             rtype=BOOT_RESOURCE_TYPE.GENERATED)

=== modified file 'src/maasserver/models/tests/test_bootresourceset.py'
--- src/maasserver/models/tests/test_bootresourceset.py	2014-09-10 16:20:31 +0000
+++ src/maasserver/models/tests/test_bootresourceset.py	2014-10-17 01:59:55 +0000
@@ -132,7 +132,7 @@
             resource_set, largefile, filename=filetype, filetype=filetype)
         self.assertEqual(0, resource_set.progress)
 
-    def test_progress_increases_from_0_to_1(self):
+    def test_progress_increases_from_0_to_100(self):
         resource = factory.make_BootResource()
         resource_set = factory.make_BootResourceSet(resource)
         filetype = BOOT_RESOURCE_FILE_TYPE.ROOT_IMAGE
@@ -149,7 +149,7 @@
             stream.write(b"a")
             current_size += 1
             self.assertAlmostEqual(
-                total_size / float(current_size),
+                100.0 * current_size / float(total_size),
                 resource_set.progress)
 
     def test_progress_accumulates_all_files(self):
@@ -174,7 +174,7 @@
                 content=content, size=total_size)
             factory.make_BootResourceFile(
                 resource_set, largefile, filename=filetype, filetype=filetype)
-        progress = final_total_size / float(final_size)
+        progress = 100.0 * final_size / float(final_total_size)
         self.assertAlmostEqual(progress, resource_set.progress)
 
     def test_complete_returns_false_for_no_files(self):

=== modified file 'src/maasserver/views/images.py'
--- src/maasserver/views/images.py	2014-10-02 18:55:37 +0000
+++ src/maasserver/views/images.py	2014-10-17 01:59:55 +0000
@@ -53,6 +53,7 @@
     BootSourceCache,
     BootSourceSelection,
     Config,
+    LargeFile,
     Node,
     )
 from maasserver.views import HelpfulDeleteView
@@ -460,14 +461,149 @@
             self.add_resource_template_attributes(resource)
         return resources
 
+    def is_hwe_resource(self, resource):
+        """Return True if the resource is an Ubuntu HWE resource."""
+        if resource.rtype != BOOT_RESOURCE_TYPE.SYNCED:
+            return False
+        if not resource.name.startswith('ubuntu/'):
+            return False
+        arch, subarch = resource.split_arch()
+        return subarch.startswith('hwe-')
+
+    def pick_latest_datetime(self, time, other_time):
+        """Return the datetime that is the latest."""
+        if time is None:
+            return other_time
+        return max([time, other_time])
+
+    def calculate_unique_size_for_resources(self, resources):
+        """Return size of all unique largefiles for the given resources."""
+        shas = set()
+        size = 0
+        for resource in resources:
+            resource_set = resource.get_latest_set()
+            if resource_set is None:
+                continue
+            for rfile in resource_set.files.all():
+                try:
+                    largefile = rfile.largefile
+                except LargeFile.DoesNotExist:
+                    continue
+                if largefile.sha256 not in shas:
+                    size += largefile.total_size
+                    shas.add(largefile.sha256)
+        return size
+
+    def are_all_resources_complete(self, resources):
+        """Return the complete status for all the given resources."""
+        for resource in resources:
+            resource_set = resource.get_latest_set()
+            if resource_set is None:
+                return False
+            if not resource_set.complete:
+                return False
+        return True
+
+    def get_last_update_for_resources(self, resources):
+        """Return the latest updated time for all resources."""
+        last_update = None
+        for resource in resources:
+            last_update = self.pick_latest_datetime(
+                last_update, resource.updated)
+            resource_set = resource.get_latest_set()
+            if resource_set is not None:
+                last_update = self.pick_latest_datetime(
+                    last_update, resource_set.updated)
+        return last_update
+
+    def get_number_of_nodes_for_resources(self, resources):
+        """Return the number of nodes used by all resources."""
+        return sum([
+            self.get_number_of_nodes_deployed_for(resource)
+            for resource in resources])
+
+    def get_progress_for_resources(self, resources):
+        """Return the overall progress for all resources."""
+        size = 0
+        total_size = 0
+        for resource in resources:
+            resource_set = resource.get_latest_set()
+            if resource_set is not None:
+                size += resource_set.size
+                total_size += resource_set.total_size
+        if size <= 0:
+            # Handle division by zero
+            return 0
+        return 100.0 * (size / float(total_size))
+
+    def hwes_to_resource(self, hwes):
+        """Convert the list of hwes into one resource to be used in the UI."""
+        # Calculate all of the values using all of the hwe resources for
+        # this combination of resources.
+        last_update = self.get_last_update_for_resources(hwes)
+        unique_size = self.calculate_unique_size_for_resources(hwes)
+        number_of_nodes = self.get_number_of_nodes_for_resources(hwes)
+        complete = self.are_all_resources_complete(hwes)
+        progress = self.get_progress_for_resources(hwes)
+
+        # Set the computed attributes on the first resource as that will
+        # be the only one returned to the UI.
+        resource = hwes[0]
+        resource.arch, resource.subarch = resource.split_arch()
+        resource.title = self.get_resource_title(resource)
+        resource.complete = complete
+        resource.size = format_size(unique_size)
+        resource.last_update = last_update
+        resource.number_of_nodes = number_of_nodes
+        resource.complete = complete
+        if not complete:
+            if progress > 0:
+                resource.status = "Downloading %3.0f%%" % progress
+                resource.downloading = True
+            else:
+                resource.status = "Queued for download"
+                resource.downloading = False
+        else:
+            # See if all the hwe resources exist on all the clusters.
+            cluster_has_hwes = any(
+                hwe in hwes for hwe in self.cluster_resources)
+            if cluster_has_hwes:
+                resource.status = "Complete"
+                resource.downloading = False
+            else:
+                resource.complete = False
+                if self.clusters_syncing:
+                    resource.status = "Syncing to clusters"
+                    resource.downloading = True
+                else:
+                    resource.status = "Waiting for clusters to sync"
+                    resource.downloading = False
+        return resource
+
+    def combine_hwe_resources(self, resources):
+        """Return a list of resources removing the duplicate hwe resources."""
+        none_hwe_resources = []
+        hwe_resources = defaultdict(list)
+        for resource in resources:
+            if not self.is_hwe_resource(resource):
+                self.add_resource_template_attributes(resource)
+                none_hwe_resources.append(resource)
+            else:
+                arch = resource.split_arch()[0]
+                key = '%s/%s' % (resource.name, arch)
+                hwe_resources[key].append(resource)
+        combined_hwes = [
+            self.hwes_to_resource(hwes)
+            for _, hwes in hwe_resources.items()
+            ]
+        return none_hwe_resources + combined_hwes
+
     def ajax(self, request, *args, **kwargs):
         """Return all resources in a json object.
 
         This is used by the image model list on the client side to update
         the status of images."""
-        resources = list(BootResource.objects.all())
-        for resource in resources:
-            self.add_resource_template_attributes(resource)
+        resources = self.combine_hwe_resources(BootResource.objects.all())
         json_resources = [
             dict(
                 id=resource.id,

=== modified file 'src/maasserver/views/tests/test_images.py'
--- src/maasserver/views/tests/test_images.py	2014-10-02 18:55:37 +0000
+++ src/maasserver/views/tests/test_images.py	2014-10-17 01:59:55 +0000
@@ -25,6 +25,7 @@
     NODE_STATUS,
     )
 from maasserver.models import (
+    BootResource,
     BootSourceCache,
     BootSourceSelection,
     Config,
@@ -37,6 +38,7 @@
     )
 from maasserver.testing.testcase import MAASServerTestCase
 from maasserver.views import images as images_view
+from maasserver.views.images import format_size
 from maastesting.matchers import (
     MockCalledOnceWith,
     MockCalledWith,
@@ -688,6 +690,195 @@
         json_resource = json_obj['resources'][0]
         self.assertEqual(number_of_nodes, json_resource['numberOfNodes'])
 
+    def test_combines_hwe_resources_into_one_resource(self):
+        self.client_log_in()
+        name = 'ubuntu/%s' % factory.make_name('series')
+        arch = factory.make_name('arch')
+        subarches = [factory.make_name('hwe') for _ in range(3)]
+        for subarch in subarches:
+            factory.make_usable_boot_resource(
+                rtype=BOOT_RESOURCE_TYPE.SYNCED,
+                name=name, architecture='%s/%s' % (arch, subarch))
+        response = self.get_images_ajax()
+        json_obj = json.loads(response.content)
+        self.assertEqual(
+            1, len(json_obj['resources']),
+            'More than one resource was returned.')
+
+    def test_combined_hwe_resource_calculates_unique_size(self):
+        self.client_log_in()
+        name = 'ubuntu/%s' % factory.make_name('series')
+        arch = factory.make_name('arch')
+        subarches = [factory.make_name('hwe') for _ in range(3)]
+        largefile = factory.make_LargeFile()
+        for subarch in subarches:
+            resource = factory.make_BootResource(
+                rtype=BOOT_RESOURCE_TYPE.SYNCED,
+                name=name, architecture='%s/%s' % (arch, subarch))
+            resource_set = factory.make_BootResourceSet(resource)
+            factory.make_BootResourceFile(resource_set, largefile)
+        response = self.get_images_ajax()
+        json_obj = json.loads(response.content)
+        json_resource = json_obj['resources'][0]
+        self.assertEqual(
+            format_size(largefile.total_size), json_resource['size'])
+
+    def test_combined_hwe_resource_calculates_number_of_nodes_deployed(self):
+        self.client_log_in()
+        osystem = 'ubuntu'
+        series = factory.make_name('series')
+        name = '%s/%s' % (osystem, series)
+        arch = factory.make_name('arch')
+        subarches = [factory.make_name('hwe') for _ in range(3)]
+        for subarch in subarches:
+            factory.make_usable_boot_resource(
+                rtype=BOOT_RESOURCE_TYPE.SYNCED,
+                name=name, architecture='%s/%s' % (arch, subarch))
+
+        number_of_nodes = random.randint(1, 4)
+        for _ in range(number_of_nodes):
+            subarch = random.choice(subarches)
+            node_architecture = '%s/%s' % (arch, subarch)
+            factory.make_Node(
+                status=NODE_STATUS.DEPLOYED,
+                osystem=osystem, distro_series=series,
+                architecture=node_architecture)
+
+        response = self.get_images_ajax()
+        json_obj = json.loads(response.content)
+        json_resource = json_obj['resources'][0]
+        self.assertEqual(number_of_nodes, json_resource['numberOfNodes'])
+
+    def test_combined_hwe_resource_calculates_complete_True(self):
+        self.client_log_in()
+        name = 'ubuntu/%s' % factory.make_name('series')
+        arch = factory.make_name('arch')
+        subarches = [factory.make_name('hwe') for _ in range(3)]
+        resources = [
+            factory.make_usable_boot_resource(
+                rtype=BOOT_RESOURCE_TYPE.SYNCED,
+                name=name, architecture='%s/%s' % (arch, subarch))
+            for subarch in subarches
+            ]
+        self.patch(
+            BootResource.objects,
+            'get_resources_matching_boot_images').return_value = resources
+        response = self.get_images_ajax()
+        json_obj = json.loads(response.content)
+        json_resource = json_obj['resources'][0]
+        self.assertTrue(json_resource['complete'])
+
+    def test_combined_hwe_resource_calculates_complete_False(self):
+        self.client_log_in()
+        name = 'ubuntu/%s' % factory.make_name('series')
+        arch = factory.make_name('arch')
+        subarches = [factory.make_name('hwe') for _ in range(3)]
+        incomplete_subarch = subarches.pop()
+        factory.make_BootResource(
+            rtype=BOOT_RESOURCE_TYPE.SYNCED,
+            name=name, architecture='%s/%s' % (arch, incomplete_subarch))
+        for subarch in subarches:
+            factory.make_usable_boot_resource(
+                rtype=BOOT_RESOURCE_TYPE.SYNCED,
+                name=name, architecture='%s/%s' % (arch, subarch))
+        response = self.get_images_ajax()
+        json_obj = json.loads(response.content)
+        json_resource = json_obj['resources'][0]
+        self.assertFalse(json_resource['complete'])
+
+    def test_combined_hwe_resource_calculates_progress(self):
+        self.client_log_in()
+        name = 'ubuntu/%s' % factory.make_name('series')
+        arch = factory.make_name('arch')
+        subarches = [factory.make_name('hwe') for _ in range(3)]
+        largefile = factory.make_LargeFile()
+        largefile.total_size = largefile.total_size * 2
+        largefile.save()
+        for subarch in subarches:
+            resource = factory.make_BootResource(
+                rtype=BOOT_RESOURCE_TYPE.SYNCED,
+                name=name, architecture='%s/%s' % (arch, subarch))
+            resource_set = factory.make_BootResourceSet(resource)
+            factory.make_BootResourceFile(resource_set, largefile)
+        response = self.get_images_ajax()
+        json_obj = json.loads(response.content)
+        json_resource = json_obj['resources'][0]
+        self.assertEqual("Downloading  50%", json_resource['status'])
+
+    def test_combined_hwe_resource_shows_queued_if_no_progress(self):
+        self.client_log_in()
+        name = 'ubuntu/%s' % factory.make_name('series')
+        arch = factory.make_name('arch')
+        subarches = [factory.make_name('hwe') for _ in range(3)]
+        largefile = factory.make_LargeFile(content="")
+        for subarch in subarches:
+            resource = factory.make_BootResource(
+                rtype=BOOT_RESOURCE_TYPE.SYNCED,
+                name=name, architecture='%s/%s' % (arch, subarch))
+            resource_set = factory.make_BootResourceSet(resource)
+            factory.make_BootResourceFile(resource_set, largefile)
+        response = self.get_images_ajax()
+        json_obj = json.loads(response.content)
+        json_resource = json_obj['resources'][0]
+        self.assertEqual("Queued for download", json_resource['status'])
+
+    def test_combined_hwe_resource_shows_complete_status(self):
+        self.client_log_in()
+        name = 'ubuntu/%s' % factory.make_name('series')
+        arch = factory.make_name('arch')
+        subarches = [factory.make_name('hwe') for _ in range(3)]
+        resources = [
+            factory.make_usable_boot_resource(
+                rtype=BOOT_RESOURCE_TYPE.SYNCED,
+                name=name, architecture='%s/%s' % (arch, subarch))
+            for subarch in subarches
+            ]
+        self.patch(
+            BootResource.objects,
+            'get_resources_matching_boot_images').return_value = resources
+        response = self.get_images_ajax()
+        json_obj = json.loads(response.content)
+        json_resource = json_obj['resources'][0]
+        self.assertEqual("Complete", json_resource['status'])
+
+    def test_combined_hwe_resource_shows_waiting_for_cluster_to_sync(self):
+        self.client_log_in()
+        name = 'ubuntu/%s' % factory.make_name('series')
+        arch = factory.make_name('arch')
+        subarches = [factory.make_name('hwe') for _ in range(3)]
+        for subarch in subarches:
+            factory.make_usable_boot_resource(
+                rtype=BOOT_RESOURCE_TYPE.SYNCED,
+                name=name, architecture='%s/%s' % (arch, subarch))
+        self.patch(
+            BootResource.objects,
+            'get_resources_matching_boot_images').return_value = []
+        response = self.get_images_ajax()
+        json_obj = json.loads(response.content)
+        json_resource = json_obj['resources'][0]
+        self.assertEqual(
+            "Waiting for clusters to sync", json_resource['status'])
+
+    def test_combined_hwe_resource_shows_clusters_syncing(self):
+        self.client_log_in()
+        name = 'ubuntu/%s' % factory.make_name('series')
+        arch = factory.make_name('arch')
+        subarches = [factory.make_name('hwe') for _ in range(3)]
+        for subarch in subarches:
+            factory.make_usable_boot_resource(
+                rtype=BOOT_RESOURCE_TYPE.SYNCED,
+                name=name, architecture='%s/%s' % (arch, subarch))
+        self.patch(
+            BootResource.objects,
+            'get_resources_matching_boot_images').return_value = []
+        self.patch(
+            images_view, 'is_import_boot_images_running').return_value = True
+        response = self.get_images_ajax()
+        json_obj = json.loads(response.content)
+        json_resource = json_obj['resources'][0]
+        self.assertEqual(
+            "Syncing to clusters", json_resource['status'])
+
 
 class TestImageDelete(MAASServerTestCase):
 

=== modified file 'src/provisioningserver/import_images/download_descriptions.py'
--- src/provisioningserver/import_images/download_descriptions.py	2014-09-30 20:54:32 +0000
+++ src/provisioningserver/import_images/download_descriptions.py	2014-10-17 01:59:55 +0000
@@ -78,14 +78,12 @@
         """Overridable from `BasicMirrorWriter`."""
         item = products_exdata(src, pedigree)
         os = get_os_from_product(item)
-        arch, subarches = item['arch'], item['subarches']
+        arch, subarch = item['arch'], item['subarch']
         release = item['release']
         label = item['label']
-        base_image = ImageSpec(os, arch, None, release, label)
+        base_image = ImageSpec(os, arch, subarch, release, label)
         compact_item = clean_up_repo_item(item)
-        for subarch in subarches.split(','):
-            self.boot_images_dict.setdefault(
-                base_image._replace(subarch=subarch), compact_item)
+        self.boot_images_dict.setdefault(base_image, compact_item)
 
 
 def value_passes_filter_list(filter_list, property_value):