Merge lp:~blake-rouse/maas/large-object-store into lp:~maas-committers/maas/trunk

[Blake Rouse]

New LargeObjectField, that stores data into postgres large object storage.

This will be used for storing the boot resource files inside of the regions db. This will allow for easy HA, once that is enabled.

[Gavin Panella (allenap)]

This looks good. I think it needs a bit more test coverage, so I'm kind of +0.7 right now.

review: Needs Fixing

[Blake Rouse (blake-rouse)]

Made the requested changes.

[Gavin Panella (allenap)]

Tip top.

review: Approve

=== modified file 'src/maasserver/fields.py'
--- src/maasserver/fields.py	2014-07-08 07:33:43 +0000
+++ src/maasserver/fields.py	2014-08-07 18:42:43 +0000
@@ -31,12 +31,15 @@
 
 from django.core.exceptions import ValidationError
 from django.core.validators import RegexValidator
+from django.db import connections
 from django.db.models import (
     BinaryField,
     Field,
     GenericIPAddressField,
+    IntegerField,
     SubfieldBase,
     )
+from django.db.models.fields.subclassing import Creator
 from django.forms import (
     CharField,
     ModelChoiceField,
@@ -93,6 +96,7 @@
         "^maasserver\.fields\.XMLField",
         "^maasserver\.fields\.EditableBinaryField",
         "^maasserver\.fields\.MAASIPAddressField",
+        "^maasserver\.fields\.LargeObjectField",
     ])
 
 
@@ -412,3 +416,114 @@
         and force a 'inet' type field.
         """
         return 'inet'
+
+
+class LargeObjectFile(object):
+    """Large object file.
+
+    Proxy the access from this object to psycopg2.
+    """
+    def __init__(self, oid=0, field=None, instance=None, block_size=(1 << 16)):
+        self.oid = oid
+        self.field = field
+        self.instance = instance
+        self.block_size = block_size
+        self._lobject = None
+
+    def __getattr__(self, name):
+        if self._lobject is None:
+            raise IOError("LargeObjectFile is not opened.")
+        return getattr(self._lobject, name)
+
+    def __enter__(self, *args, **kwargs):
+        return self
+
+    def __exit__(self, *args, **kwargs):
+        self.close()
+
+    def __iter__(self):
+        return self
+
+    def open(self, mode="rwb", new_file=None, using="default"):
+        """Opens the internal large object instance."""
+        connection = connections[using]
+        self._lobject = connection.connection.lobject(
+            self.oid, mode, 0, new_file)
+        self.oid = self._lobject.oid
+        return self
+
+    def unlink(self):
+        """Removes the large object."""
+        if self._lobject is None:
+            # Need to open the lobject so we get a reference to it in the
+            # database, to perform the unlink.
+            self.open()
+            self.close()
+        self._lobject.unlink()
+        self._lobject = None
+        self.oid = 0
+
+    def next(self):
+        r = self.read(self.block_size)
+        if len(r) == 0:
+            raise StopIteration
+        return r
+
+
+class LargeObjectDescriptor(Creator):
+    """LargeObjectField descriptor."""
+
+    def __set__(self, instance, value):
+        value = self.field.to_python(value)
+        if value is not None:
+            if not isinstance(value, LargeObjectFile):
+                value = LargeObjectFile(value, self.field, instance)
+        instance.__dict__[self.field.name] = value
+
+
+class LargeObjectField(IntegerField):
+    """A field that stores large amounts of data into postgres large object
+    storage.
+
+    Internally the field on the model is an `oid` field, that returns a proxy
+    to the referenced large object.
+    """
+
+    def __init__(self, *args, **kwargs):
+        self.block_size = kwargs.pop('block_size', 1 << 16)
+        super(LargeObjectField, self).__init__(*args, **kwargs)
+
+    def db_type(self, connection):
+        """Returns the database column data type for LargeObjectField."""
+        # oid is the column type postgres uses to reference a large object
+        return 'oid'
+
+    def contribute_to_class(self, cls, name):
+        """Set the descriptor for the large object."""
+        super(LargeObjectField, self).contribute_to_class(cls, name)
+        setattr(cls, self.name, LargeObjectDescriptor(self))
+
+    def get_db_prep_value(self, value, connection=None, prepared=False):
+        """python -> db: `oid` value"""
+        if value is None:
+            return None
+        if isinstance(value, LargeObjectFile):
+            if value.oid > 0:
+                return value.oid
+            raise AssertionError(
+                "LargeObjectFile's oid must be greater than 0.")
+        raise AssertionError(
+            "Invalid LargeObjectField value (expected LargeObjectFile): '%s'"
+            % repr(value))
+
+    def to_python(self, value):
+        """db -> python: `LargeObjectFile`"""
+        if value is None:
+            return None
+        elif isinstance(value, LargeObjectFile):
+            return value
+        elif isinstance(value, (int, long)):
+            return LargeObjectFile(value, self, self.model, self.block_size)
+        raise AssertionError(
+            "Invalid LargeObjectField value (expected integer): '%s'"
+            % repr(value))

=== modified file 'src/maasserver/tests/models.py'
--- src/maasserver/tests/models.py	2014-07-08 07:52:05 +0000
+++ src/maasserver/tests/models.py	2014-08-07 18:42:43 +0000
@@ -25,6 +25,7 @@
     )
 from maasserver.fields import (
     JSONObjectField,
+    LargeObjectField,
     MAASIPAddressField,
     XMLField,
     )
@@ -73,3 +74,8 @@
 
 class MAASIPAddressFieldModel(Model):
     ip_address = MAASIPAddressField()
+
+
+class LargeObjectFieldModel(Model):
+    name = CharField(max_length=255, unique=False)
+    large_object = LargeObjectField(block_size=10)

=== modified file 'src/maasserver/tests/test_api_node.py'
--- src/maasserver/tests/test_api_node.py	2014-08-05 10:13:38 +0000
+++ src/maasserver/tests/test_api_node.py	2014-08-07 18:42:43 +0000
@@ -18,7 +18,6 @@
 from cStringIO import StringIO
 import httplib
 import json
-from netaddr import IPAddress
 import sys
 
 import bson
@@ -53,6 +52,7 @@
     NodeUserData,
     )
 from metadataserver.nodeinituser import get_node_init_user
+from netaddr import IPAddress
 from provisioningserver.utils import map_enum
 
 

=== modified file 'src/maasserver/tests/test_fields.py'
--- src/maasserver/tests/test_fields.py	2014-07-16 14:12:13 +0000
+++ src/maasserver/tests/test_fields.py	2014-08-07 18:42:43 +0000
@@ -27,6 +27,8 @@
 from maasserver.enum import NODEGROUPINTERFACE_MANAGEMENT
 from maasserver.fields import (
     EditableBinaryField,
+    LargeObjectField,
+    LargeObjectFile,
     MAC,
     NodeGroupFormField,
     register_mac_type,
@@ -42,10 +44,13 @@
 from maasserver.testing.testcase import MAASServerTestCase
 from maasserver.tests.models import (
     JSONFieldModel,
+    LargeObjectFieldModel,
     MAASIPAddressFieldModel,
     XMLFieldModel,
     )
 from maastesting.djangotestcase import TestModelMixin
+from maastesting.matchers import MockCalledOnceWith
+from psycopg2 import OperationalError
 from psycopg2.extensions import ISQLQuote
 
 
@@ -401,3 +406,109 @@
         results = MAASIPAddressFieldModel.objects.filter(
             ip_address__lte='192.0.2.100')
         self.assertItemsEqual([ip_object], results)
+
+
+class TestLargeObjectField(TestModelMixin, MAASServerTestCase):
+
+    app = 'maasserver.tests'
+
+    def test_stores_data(self):
+        data = factory.make_string()
+        test_name = factory.make_name('name')
+        test_instance = LargeObjectFieldModel(name=test_name)
+        large_object = LargeObjectFile()
+        with large_object.open('wb') as stream:
+            stream.write(data)
+        test_instance.large_object = large_object
+        test_instance.save()
+        test_instance = LargeObjectFieldModel.objects.get(name=test_name)
+        with test_instance.large_object.open('rb') as stream:
+            saved_data = stream.read()
+        self.assertEqual(data, saved_data)
+
+    def test_with_exit_calls_close(self):
+        data = factory.make_string()
+        large_object = LargeObjectFile()
+        with large_object.open('wb') as stream:
+            self.addCleanup(large_object.close)
+            mock_close = self.patch(large_object, 'close')
+            stream.write(data)
+        self.assertThat(mock_close, MockCalledOnceWith())
+
+    def test_unlink(self):
+        data = factory.make_string()
+        large_object = LargeObjectFile()
+        with large_object.open('wb') as stream:
+            stream.write(data)
+        oid = large_object.oid
+        large_object.unlink()
+        self.assertEqual(0, large_object.oid)
+        self.assertRaises(
+            OperationalError,
+            connection.connection.lobject, oid)
+
+    def test_interates_on_block_size(self):
+        # String size is multiple of block_size in the testing model
+        data = factory.make_string(10 * 2)
+        test_name = factory.make_name('name')
+        test_instance = LargeObjectFieldModel(name=test_name)
+        large_object = LargeObjectFile()
+        with large_object.open('wb') as stream:
+            stream.write(data)
+        test_instance.large_object = large_object
+        test_instance.save()
+        test_instance = LargeObjectFieldModel.objects.get(name=test_name)
+        with test_instance.large_object.open('rb') as stream:
+            offset = 0
+            for block in stream:
+                self.assertEqual(data[offset:offset + 10], block)
+                offset += 10
+
+    def test_get_db_prep_value_returns_None_when_value_None(self):
+        field = LargeObjectField()
+        self.assertEqual(None, field.get_db_prep_value(None))
+
+    def test_get_db_prep_value_returns_oid_when_value_LargeObjectFile(self):
+        oid = randint(1, 100)
+        field = LargeObjectField()
+        obj_file = LargeObjectFile()
+        obj_file.oid = oid
+        self.assertEqual(oid, field.get_db_prep_value(obj_file))
+
+    def test_get_db_prep_value_raises_error_when_oid_less_than_zero(self):
+        oid = randint(-100, 0)
+        field = LargeObjectField()
+        obj_file = LargeObjectFile()
+        obj_file.oid = oid
+        self.assertRaises(AssertionError, field.get_db_prep_value, obj_file)
+
+    def test_get_db_prep_value_raises_error_when_not_LargeObjectFile(self):
+        field = LargeObjectField()
+        self.assertRaises(
+            AssertionError, field.get_db_prep_value, factory.make_string())
+
+    def test_to_python_returns_None_when_value_None(self):
+        field = LargeObjectField()
+        self.assertEqual(None, field.to_python(None))
+
+    def test_to_python_returns_value_when_value_LargeObjectFile(self):
+        field = LargeObjectField()
+        obj_file = LargeObjectFile()
+        self.assertEqual(obj_file, field.to_python(obj_file))
+
+    def test_to_python_returns_LargeObjectFile_when_value_int(self):
+        oid = randint(1, 100)
+        field = LargeObjectField()
+        obj_file = field.to_python(oid)
+        self.assertEqual(oid, obj_file.oid)
+
+    def test_to_python_returns_LargeObjectFile_when_value_long(self):
+        oid = long(randint(1, 100))
+        field = LargeObjectField()
+        obj_file = field.to_python(oid)
+        self.assertEqual(oid, obj_file.oid)
+
+    def test_to_python_raises_error_when_not_valid_type(self):
+        field = LargeObjectField()
+        self.assertRaises(
+            AssertionError, field.to_python, factory.make_string())