Merge lp:~blake-rouse/maas/fix-1604465-2.0 into lp:maas/2.0

[Blake Rouse]

Add size to the largefile model. Add migration that fills in the size on all largefiles that already exist in MAAS. Update the current size field when images are imported. Use aggragated queries in boot resource code to speep up the database queries. Adjust image status interval check to every 30 seconds from 10 seconds.

Backport: https://code.launchpad.net/~blake-rouse/maas/fix-1604465/+merge/300655

[Blake Rouse (blake-rouse)]

Self-approving backport.

review: Approve

=== modified file 'src/maasserver/api/boot_resources.py'
--- src/maasserver/api/boot_resources.py	2016-04-12 17:36:35 +0000
+++ src/maasserver/api/boot_resources.py	2016-07-21 16:44:22 +0000
@@ -300,6 +300,8 @@
                     "Too much data recieved.")
 
             stream.write(data)
+            rfile.largefile.size = current_size + size
+            rfile.largefile.save()
 
         if rfile.largefile.complete:
             if not rfile.largefile.valid:

=== modified file 'src/maasserver/bootresources.py'
--- src/maasserver/bootresources.py	2016-07-07 21:57:57 +0000
+++ src/maasserver/bootresources.py	2016-07-21 16:44:22 +0000
@@ -669,6 +669,10 @@
             {'sha256': rfile.largefile.sha256})
         maaslog.debug("Finalizing boot image %s.", ident)
 
+        # Ensure that the size of the largefile starts at zero.
+        rfile.largefile.size = 0
+        rfile.largefile.save(update_fields=['size'])
+
         # Write the contents into the database, while calculating the sha256
         # hash for the read data.
         with rfile.largefile.content.open('wb') as stream:
@@ -676,7 +680,10 @@
                 buf = reader.read(self.read_size)
                 stream.write(buf)
                 cksummer.update(buf)
-                if len(buf) != self.read_size:
+                buf_len = len(buf)
+                rfile.largefile.size += buf_len
+                rfile.largefile.save(update_fields=['size'])
+                if buf_len != self.read_size:
                     break
 
         if not cksummer.check():

=== added file 'src/maasserver/migrations/builtin/maasserver/0067_add_size_to_largefile.py'
--- src/maasserver/migrations/builtin/maasserver/0067_add_size_to_largefile.py	1970-01-01 00:00:00 +0000
+++ src/maasserver/migrations/builtin/maasserver/0067_add_size_to_largefile.py	2016-07-21 16:44:22 +0000
@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import (
+    migrations,
+    models,
+)
+
+
+def get_size_of_content(large_file):
+    with self.content.open('rb') as stream:
+        stream.seek(0, os.SEEK_END)
+        return stream.tell()
+
+
+def set_size_on_all_largefiles(apps, schema_editor):
+    LargeFile = apps.get_model("maasserver", "LargeFile")
+    for large_file in LargeFile.objects.all():
+        large_file.size = get_size_of_content(large_file)
+        large_file.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('maasserver', '0066_allow_squashfs'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='largefile',
+            name='size',
+            field=models.BigIntegerField(default=0, editable=False),
+        ),
+        migrations.RunPython(set_size_on_all_largefiles),
+    ]

=== modified file 'src/maasserver/models/bootresource.py'
--- src/maasserver/models/bootresource.py	2016-04-12 17:36:35 +0000
+++ src/maasserver/models/bootresource.py	2016-07-21 16:44:22 +0000
@@ -10,8 +10,10 @@
 from django.core.exceptions import ValidationError
 from django.db.models import (
     CharField,
+    Count,
     IntegerField,
     Manager,
+    Sum,
 )
 from maasserver import DefaultMeta
 from maasserver.enum import (
@@ -381,8 +383,13 @@
     def get_latest_complete_set(self):
         """Return latest `BootResourceSet` where all `BootResouceFile`'s
         are complete."""
-        for resource_set in self.sets.order_by('id').reverse():
-            if resource_set.complete:
+        resource_sets = self.sets.order_by('id').annotate(
+            files_count=Count('files__id'),
+            files_size=Sum('files__largefile__size'),
+            files_total_size=Sum('files__largefile__total_size'))
+        for resource_set in resource_sets.reverse():
+            if (resource_set.files_count > 0 and
+                    resource_set.files_size == resource_set.files_total_size):
                 return resource_set
         return None
 

=== modified file 'src/maasserver/models/bootresourceset.py'
--- src/maasserver/models/bootresourceset.py	2016-07-07 21:57:57 +0000
+++ src/maasserver/models/bootresourceset.py	2016-07-21 16:44:22 +0000
@@ -10,6 +10,7 @@
 from django.db.models import (
     CharField,
     ForeignKey,
+    Sum,
 )
 from maasserver import DefaultMeta
 from maasserver.enum import BOOT_RESOURCE_FILE_TYPE
@@ -89,32 +90,41 @@
     @property
     def total_size(self):
         """Total amount of space this set will consume."""
-        return sum(
-            resource_file.largefile.total_size
-            for resource_file in self.files.all())
+        total_size = self.files.all().aggregate(
+            total_size=Sum('largefile__total_size'))['total_size']
+        if total_size is None:
+            total_size = 0
+        return total_size
 
     @property
     def size(self):
         """Amount of space this set currently consumes."""
-        return sum(
-            resource_file.largefile.size
-            for resource_file in self.files.all())
+        size = self.files.all().aggregate(size=Sum('largefile__size'))['size']
+        if size is None:
+            size = 0
+        return size
 
     @property
     def progress(self):
         """Percentage complete for all files in the set."""
-        size = self.size
-        if size <= 0:
+        size_info = self.files.all().aggregate(
+            total_size=Sum('largefile__total_size'),
+            size=Sum('largefile__size'))
+        if size_info['size'] is None:
+            size_info['size'] = 0
+        if size_info['total_size'] is None:
+            size_info['total_size'] = 0
+        if size_info['size'] <= 0:
             # Handle division by zero
             return 0
-        return 100.0 * size / float(self.total_size)
+        return 100.0 * size_info['size'] / float(size_info['total_size'])
 
     @property
     def complete(self):
         """True if all files in the set are complete."""
         if not self.files.exists():
             return False
-        for resource_file in self.files.all():
-            if not resource_file.largefile.complete:
-                return False
-        return True
+        size_info = self.files.all().aggregate(
+            total_size=Sum('largefile__total_size'),
+            size=Sum('largefile__size'))
+        return size_info['total_size'] == size_info['size']

=== modified file 'src/maasserver/models/largefile.py'
--- src/maasserver/models/largefile.py	2016-05-12 19:07:37 +0000
+++ src/maasserver/models/largefile.py	2016-07-21 16:44:22 +0000
@@ -8,7 +8,6 @@
 ]
 
 import hashlib
-import os
 
 from django.db.models import (
     BigIntegerField,
@@ -72,7 +71,7 @@
                 objstream.write(data)
                 length += len(data)
         return self.create(
-            sha256=hexdigest, total_size=length, content=objfile)
+            sha256=hexdigest, size=length, total_size=length, content=objfile)
 
 
 class LargeFile(CleanSave, TimestampedModel):
@@ -85,6 +84,7 @@
     process by only saving unique files.
 
     :ivar sha256: Calculated SHA256 value of `content`.
+    :ivar size: Current size of `content`.
     :ivar total_size: Final size of `content`. The data might currently
         be saving, so total_size could be larger than `size`. `size` should
         never be larger than `total_size`.
@@ -98,6 +98,8 @@
 
     sha256 = CharField(max_length=64, unique=True, editable=False)
 
+    size = BigIntegerField(default=0, editable=False)
+
     total_size = BigIntegerField(editable=False)
 
     # content is stored directly in the database, in the large object storage.
@@ -108,14 +110,6 @@
         return "<LargeFile size=%d sha256=%s>" % (self.total_size, self.sha256)
 
     @property
-    def size(self):
-        """Size of content."""
-        with self.content.open('rb') as stream:
-            stream.seek(0, os.SEEK_END)
-            size = stream.tell()
-        return size
-
-    @property
     def progress(self):
         """Percentage of `content` saved."""
         if self.size <= 0:

=== modified file 'src/maasserver/models/tests/test_bootresourceset.py'
--- src/maasserver/models/tests/test_bootresourceset.py	2016-07-07 21:57:57 +0000
+++ src/maasserver/models/tests/test_bootresourceset.py	2016-07-21 16:44:22 +0000
@@ -135,6 +135,8 @@
         self.assertEqual(0, resource_set.progress)
         for _ in range(total_size):
             stream.write(b"a")
+            largefile.size += 1
+            largefile.save()
             current_size += 1
             self.assertAlmostEqual(
                 100.0 * current_size / float(total_size),

=== modified file 'src/maasserver/models/tests/test_largefile.py'
--- src/maasserver/models/tests/test_largefile.py	2016-05-12 19:07:37 +0000
+++ src/maasserver/models/tests/test_largefile.py	2016-07-21 16:44:22 +0000
@@ -68,6 +68,8 @@
             written_content = stream.read()
         self.assertEqual(
             content, written_content)
+        self.assertEqual(
+            len(content), largefile.size)
 
 
 class TestLargeFile(MAASServerTestCase):
@@ -88,13 +90,6 @@
             data = stream.read()
         self.assertEqual(content, data)
 
-    def test_size(self):
-        size = randint(512, 1024)
-        total_size = randint(1025, 2048)
-        content = factory.make_bytes(size=size)
-        largefile = factory.make_LargeFile(content, size=total_size)
-        self.assertEqual(size, largefile.size)
-
     def test_progress(self):
         size = randint(512, 1024)
         total_size = randint(1025, 2048)

=== modified file 'src/maasserver/static/js/angular/directives/controller_image_status.js'
--- src/maasserver/static/js/angular/directives/controller_image_status.js	2016-05-25 20:27:50 +0000
+++ src/maasserver/static/js/angular/directives/controller_image_status.js	2016-07-21 16:44:22 +0000
@@ -11,6 +11,9 @@
         $timeout, $interval, ControllersManager) {
      var self = this;
 
+     // How often to check the sync status of a controller in seconds.
+     var CHECK_INTERVAL = 30;
+
      // List of controllers that need to have the image status updated.
      this.controllers = [];
 
@@ -67,7 +70,7 @@
              self.startTimeout = undefined;
              self.runningInterval = $interval(function() {
                  self.updateStatuses();
-             }, 10 * 1000);
+             }, CHECK_INTERVAL * 1000);
              self.updateStatuses();
          }, 100);
      };

=== modified file 'src/maasserver/testing/factory.py'
--- src/maasserver/testing/factory.py	2016-07-07 21:57:57 +0000
+++ src/maasserver/testing/factory.py	2016-07-21 16:44:22 +0000
@@ -1320,7 +1320,8 @@
         with largeobject.open('wb') as stream:
             stream.write(content)
         return LargeFile.objects.create(
-            sha256=sha256, total_size=size, content=largeobject)
+            sha256=sha256, size=len(content),
+            total_size=size, content=largeobject)
 
     def make_BootResource(self, rtype=None, name=None, architecture=None,
                           extra=None, kflavor=None):

=== modified file 'src/maasserver/tests/test_bootresources.py'
--- src/maasserver/tests/test_bootresources.py	2016-07-07 21:57:57 +0000
+++ src/maasserver/tests/test_bootresources.py	2016-07-21 16:44:22 +0000
@@ -810,6 +810,9 @@
         with rfile.largefile.content.open('rb') as stream:
             written_data = stream.read()
         self.assertEqual(content, written_data)
+        rfile.largefile = reload_object(rfile.largefile)
+        self.assertEqual(rfile.largefile.size, len(written_data))
+        self.assertEqual(rfile.largefile.size, rfile.largefile.total_size)
 
     @skip(
         "XXX blake_r: Skipped because it causes the test that runs after this "