Merge lp:~jtv/maas/migrate-filestorage into lp:~maas-committers/maas/trunk

[Jeroen T. Vermeulen]

Model migration: FileStorage.

As per the migration plan, which is proceeding uneventfully (yay). This moves FileStorage and its manager into their own module. And I even remembered to “bzr add” the new module this time.

Due to land on the 18th.

Jeroen

[Gavin Panella (allenap)]

review: Approve

=== modified file 'src/maasserver/models/__init__.py'
--- src/maasserver/models/__init__.py	2012-05-11 08:02:22 +0000
+++ src/maasserver/models/__init__.py	2012-05-17 09:20:23 +0000
@@ -20,6 +20,7 @@
     "get_auth_tokens",
     "get_db_state",
     "get_html_display_for_key",
+    "logger",
     "Config",
     "FileStorage",
     "NODE_TRANSITIONS",
@@ -31,12 +32,10 @@
 
 import binascii
 from cgi import escape
-from errno import ENOENT
 from logging import getLogger
 import os
 import re
 from string import whitespace
-import time
 from uuid import uuid1
 
 from django.conf import settings
@@ -47,12 +46,9 @@
     PermissionDenied,
     ValidationError,
     )
-from django.core.files.base import ContentFile
-from django.core.files.storage import FileSystemStorage
 from django.db import connection
 from django.db.models import (
     CharField,
-    FileField,
     ForeignKey,
     IntegerField,
     Manager,
@@ -82,6 +78,7 @@
 from maasserver.fields import MACAddressField
 from maasserver.models.cleansave import CleanSave
 from maasserver.models.config import Config
+from maasserver.models.filestorage import FileStorage
 from maasserver.models.timestampedmodel import TimestampedModel
 from metadataserver import nodeinituser
 from piston.models import (
@@ -880,136 +877,6 @@
         return mark_safe(get_html_display_for_key(self.key, MAX_KEY_DISPLAY))
 
 
-# Due for model migration on 2012-05-18
-class FileStorageManager(Manager):
-    """Manager for `FileStorage` objects.
-
-    Store files by calling `save_file`.  No two `FileStorage` objects can
-    have the same filename at the same time.  Writing new data to a file
-    whose name is already in use, replaces its `FileStorage` with one
-    pointing to the new data.
-
-    Underneath, however, the storage layer will keep the old version of the
-    file around indefinitely.  Thus, if the overwriting transaction rolls
-    back, it may leave the new file as garbage on the filesystem; but the
-    original file will not be affected.  Also, any ongoing reads from the
-    old file will continue without iterruption.
-    """
-    # The time, in seconds, that an unreferenced file is allowed to
-    # persist in order to satisfy ongoing requests.
-    grace_time = 12 * 60 * 60
-
-    def get_existing_storage(self, filename):
-        """Return an existing `FileStorage` of this name, or None."""
-        existing_storage = self.filter(filename=filename)
-        if len(existing_storage) == 0:
-            return None
-        elif len(existing_storage) == 1:
-            return existing_storage[0]
-        else:
-            raise AssertionError(
-                "There are %d files called '%s'."
-                % (len(existing_storage), filename))
-
-    def save_file(self, filename, file_object):
-        """Save the file to the filesystem and persist to the database.
-
-        The file will end up in MEDIA_ROOT/storage/
-
-        If a file of that name already existed, it will be replaced by the
-        new contents.
-        """
-        # This probably ought to read in chunks but large files are
-        # not expected.  Also note that uploading a file with the same
-        # name as an existing one will cause that file to be written
-        # with a new generated name, and the old one remains where it
-        # is.  See https://code.djangoproject.com/ticket/6157 - the
-        # Django devs consider deleting things dangerous ... ha.
-        # HOWEVER - this operation would need to be atomic anyway so
-        # it's safest left how it is for now (reads can overlap with
-        # writes from Juju).
-        content = ContentFile(file_object.read())
-
-        storage = self.get_existing_storage(filename)
-        if storage is None:
-            storage = FileStorage(filename=filename)
-        storage.data.save(filename, content)
-        return storage
-
-    def list_stored_files(self):
-        """Find the files stored in the filesystem."""
-        dirs, files = FileStorage.storage.listdir(FileStorage.upload_dir)
-        return [
-            os.path.join(FileStorage.upload_dir, filename)
-            for filename in files]
-
-    def list_referenced_files(self):
-        """Find the names of files that are referenced from `FileStorage`.
-
-        :return: All file paths within MEDIA ROOT (relative to MEDIA_ROOT)
-            that have `FileStorage` entries referencing them.
-        :rtype: frozenset
-        """
-        return frozenset(
-            file_storage.data.name
-            for file_storage in self.all())
-
-    def is_old(self, storage_filename):
-        """Is the named file in the filesystem storage old enough to be dead?
-
-        :param storage_filename: The name under which the file is stored in
-            the filesystem, relative to MEDIA_ROOT.  This need not be the
-            same name as its filename as stored in the `FileStorage` object.
-            It includes the name of the upload directory.
-        """
-        file_path = os.path.join(settings.MEDIA_ROOT, storage_filename)
-        mtime = os.stat(file_path).st_mtime
-        expiry = mtime + self.grace_time
-        return expiry <= time.time()
-
-    def collect_garbage(self):
-        """Clean up stored files that are no longer accessible."""
-        try:
-            stored_files = self.list_stored_files()
-        except OSError as e:
-            if e.errno != ENOENT:
-                raise
-            logger.info(
-                "Upload directory does not exist yet.  "
-                "Skipping garbage collection.")
-            return
-        referenced_files = self.list_referenced_files()
-        for path in stored_files:
-            if path not in referenced_files and self.is_old(path):
-                FileStorage.storage.delete(path)
-
-
-# Due for model migration on 2012-05-18
-class FileStorage(CleanSave, Model):
-    """A simple file storage keyed on file name.
-
-    :ivar filename: A unique file name to use for the data being stored.
-    :ivar data: The file's actual data.
-    """
-
-    class Meta(DefaultMeta):
-        """Needed for South to recognize this model."""
-
-    storage = FileSystemStorage()
-
-    upload_dir = "storage"
-
-    # Unix filenames can be longer than this (e.g. 255 bytes), but leave
-    # some extra room for the full path, as well as a versioning suffix.
-    filename = CharField(max_length=200, unique=True, editable=False)
-    data = FileField(upload_to=upload_dir, storage=storage, max_length=255)
-
-    objects = FileStorageManager()
-
-    def __unicode__(self):
-        return self.filename
-
-
 # Register the models in the admin site.
 admin.site.register(Consumer)
 admin.site.register(Config)

=== added file 'src/maasserver/models/filestorage.py'
--- src/maasserver/models/filestorage.py	1970-01-01 00:00:00 +0000
+++ src/maasserver/models/filestorage.py	2012-05-17 09:20:23 +0000
@@ -0,0 +1,163 @@
+# Copyright 2012 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Storage for uploaded files."""
+
+from __future__ import (
+    absolute_import,
+    print_function,
+    unicode_literals,
+    )
+
+__metaclass__ = type
+__all__ = [
+    'FileStorage',
+    ]
+
+
+from errno import ENOENT
+import os
+import time
+
+from django.conf import settings
+from django.core.files.base import ContentFile
+from django.core.files.storage import FileSystemStorage
+from django.db.models import (
+    CharField,
+    FileField,
+    Manager,
+    Model,
+    )
+from maasserver import DefaultMeta
+from maasserver.models.cleansave import CleanSave
+
+
+class FileStorageManager(Manager):
+    """Manager for `FileStorage` objects.
+
+    Store files by calling `save_file`.  No two `FileStorage` objects can
+    have the same filename at the same time.  Writing new data to a file
+    whose name is already in use, replaces its `FileStorage` with one
+    pointing to the new data.
+
+    Underneath, however, the storage layer will keep the old version of the
+    file around indefinitely.  Thus, if the overwriting transaction rolls
+    back, it may leave the new file as garbage on the filesystem; but the
+    original file will not be affected.  Also, any ongoing reads from the
+    old file will continue without iterruption.
+    """
+    # The time, in seconds, that an unreferenced file is allowed to
+    # persist in order to satisfy ongoing requests.
+    grace_time = 12 * 60 * 60
+
+    def get_existing_storage(self, filename):
+        """Return an existing `FileStorage` of this name, or None."""
+        existing_storage = self.filter(filename=filename)
+        if len(existing_storage) == 0:
+            return None
+        elif len(existing_storage) == 1:
+            return existing_storage[0]
+        else:
+            raise AssertionError(
+                "There are %d files called '%s'."
+                % (len(existing_storage), filename))
+
+    def save_file(self, filename, file_object):
+        """Save the file to the filesystem and persist to the database.
+
+        The file will end up in MEDIA_ROOT/storage/
+
+        If a file of that name already existed, it will be replaced by the
+        new contents.
+        """
+        # This probably ought to read in chunks but large files are
+        # not expected.  Also note that uploading a file with the same
+        # name as an existing one will cause that file to be written
+        # with a new generated name, and the old one remains where it
+        # is.  See https://code.djangoproject.com/ticket/6157 - the
+        # Django devs consider deleting things dangerous ... ha.
+        # HOWEVER - this operation would need to be atomic anyway so
+        # it's safest left how it is for now (reads can overlap with
+        # writes from Juju).
+        content = ContentFile(file_object.read())
+
+        storage = self.get_existing_storage(filename)
+        if storage is None:
+            storage = FileStorage(filename=filename)
+        storage.data.save(filename, content)
+        return storage
+
+    def list_stored_files(self):
+        """Find the files stored in the filesystem."""
+        dirs, files = FileStorage.storage.listdir(FileStorage.upload_dir)
+        return [
+            os.path.join(FileStorage.upload_dir, filename)
+            for filename in files]
+
+    def list_referenced_files(self):
+        """Find the names of files that are referenced from `FileStorage`.
+
+        :return: All file paths within MEDIA ROOT (relative to MEDIA_ROOT)
+            that have `FileStorage` entries referencing them.
+        :rtype: frozenset
+        """
+        return frozenset(
+            file_storage.data.name
+            for file_storage in self.all())
+
+    def is_old(self, storage_filename):
+        """Is the named file in the filesystem storage old enough to be dead?
+
+        :param storage_filename: The name under which the file is stored in
+            the filesystem, relative to MEDIA_ROOT.  This need not be the
+            same name as its filename as stored in the `FileStorage` object.
+            It includes the name of the upload directory.
+        """
+        file_path = os.path.join(settings.MEDIA_ROOT, storage_filename)
+        mtime = os.stat(file_path).st_mtime
+        expiry = mtime + self.grace_time
+        return expiry <= time.time()
+
+    def collect_garbage(self):
+        """Clean up stored files that are no longer accessible."""
+        # Avoid circular imports.
+        from maasserver.models import logger
+
+        try:
+            stored_files = self.list_stored_files()
+        except OSError as e:
+            if e.errno != ENOENT:
+                raise
+            logger.info(
+                "Upload directory does not exist yet.  "
+                "Skipping garbage collection.")
+            return
+        referenced_files = self.list_referenced_files()
+        for path in stored_files:
+            if path not in referenced_files and self.is_old(path):
+                FileStorage.storage.delete(path)
+
+
+class FileStorage(CleanSave, Model):
+    """A simple file storage keyed on file name.
+
+    :ivar filename: A unique file name to use for the data being stored.
+    :ivar data: The file's actual data.
+    """
+
+    class Meta(DefaultMeta):
+        """Needed for South to recognize this model."""
+
+    storage = FileSystemStorage()
+
+    upload_dir = "storage"
+
+    # Unix filenames can be longer than this (e.g. 255 bytes), but leave
+    # some extra room for the full path, as well as a versioning suffix.
+    filename = CharField(max_length=200, unique=True, editable=False)
+    data = FileField(upload_to=upload_dir, storage=storage, max_length=255)
+
+    objects = FileStorageManager()
+
+    def __unicode__(self):
+        return self.filename