Merge lp:~blake-rouse/maas/large-object-store into lp:maas/trunk

Proposed by Blake Rouse on 2014-08-07
Status: Merged
Approved by: Blake Rouse on 2014-08-07
Approved revision: 2659
Merged at revision: 2658
Proposed branch: lp:~blake-rouse/maas/large-object-store
Merge into: lp:maas/trunk
Diff against target: 320 lines (+233/-1)
4 files modified
src/maasserver/fields.py (+115/-0)
src/maasserver/tests/models.py (+6/-0)
src/maasserver/tests/test_api_node.py (+1/-1)
src/maasserver/tests/test_fields.py (+111/-0)
To merge this branch: bzr merge lp:~blake-rouse/maas/large-object-store
Reviewer Review Type Date Requested Status
Gavin Panella (community) 2014-08-07 Approve on 2014-08-07
Review via email: mp+229938@code.launchpad.net

Commit message

New LargeObjectField, that stores data into postgres large object storage.

Description of the change

This will be used for storing the boot resource files inside of the regions db. This will allow for easy HA, once that is enabled.

To post a comment you must log in.
Gavin Panella (allenap) wrote :

This looks good. I think it needs a bit more test coverage, so I'm kind of +0.7 right now.

review: Needs Fixing
Blake Rouse (blake-rouse) wrote :

Made the requested changes.

Gavin Panella (allenap) :
2659. By Blake Rouse on 2014-08-07

Fixes from code review.

Gavin Panella (allenap) wrote :

Tip top.

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'src/maasserver/fields.py'
2--- src/maasserver/fields.py 2014-07-08 07:33:43 +0000
3+++ src/maasserver/fields.py 2014-08-07 18:42:43 +0000
4@@ -31,12 +31,15 @@
5
6 from django.core.exceptions import ValidationError
7 from django.core.validators import RegexValidator
8+from django.db import connections
9 from django.db.models import (
10 BinaryField,
11 Field,
12 GenericIPAddressField,
13+ IntegerField,
14 SubfieldBase,
15 )
16+from django.db.models.fields.subclassing import Creator
17 from django.forms import (
18 CharField,
19 ModelChoiceField,
20@@ -93,6 +96,7 @@
21 "^maasserver\.fields\.XMLField",
22 "^maasserver\.fields\.EditableBinaryField",
23 "^maasserver\.fields\.MAASIPAddressField",
24+ "^maasserver\.fields\.LargeObjectField",
25 ])
26
27
28@@ -412,3 +416,114 @@
29 and force a 'inet' type field.
30 """
31 return 'inet'
32+
33+
34+class LargeObjectFile(object):
35+ """Large object file.
36+
37+ Proxy the access from this object to psycopg2.
38+ """
39+ def __init__(self, oid=0, field=None, instance=None, block_size=(1 << 16)):
40+ self.oid = oid
41+ self.field = field
42+ self.instance = instance
43+ self.block_size = block_size
44+ self._lobject = None
45+
46+ def __getattr__(self, name):
47+ if self._lobject is None:
48+ raise IOError("LargeObjectFile is not opened.")
49+ return getattr(self._lobject, name)
50+
51+ def __enter__(self, *args, **kwargs):
52+ return self
53+
54+ def __exit__(self, *args, **kwargs):
55+ self.close()
56+
57+ def __iter__(self):
58+ return self
59+
60+ def open(self, mode="rwb", new_file=None, using="default"):
61+ """Opens the internal large object instance."""
62+ connection = connections[using]
63+ self._lobject = connection.connection.lobject(
64+ self.oid, mode, 0, new_file)
65+ self.oid = self._lobject.oid
66+ return self
67+
68+ def unlink(self):
69+ """Removes the large object."""
70+ if self._lobject is None:
71+ # Need to open the lobject so we get a reference to it in the
72+ # database, to perform the unlink.
73+ self.open()
74+ self.close()
75+ self._lobject.unlink()
76+ self._lobject = None
77+ self.oid = 0
78+
79+ def next(self):
80+ r = self.read(self.block_size)
81+ if len(r) == 0:
82+ raise StopIteration
83+ return r
84+
85+
86+class LargeObjectDescriptor(Creator):
87+ """LargeObjectField descriptor."""
88+
89+ def __set__(self, instance, value):
90+ value = self.field.to_python(value)
91+ if value is not None:
92+ if not isinstance(value, LargeObjectFile):
93+ value = LargeObjectFile(value, self.field, instance)
94+ instance.__dict__[self.field.name] = value
95+
96+
97+class LargeObjectField(IntegerField):
98+ """A field that stores large amounts of data into postgres large object
99+ storage.
100+
101+ Internally the field on the model is an `oid` field, that returns a proxy
102+ to the referenced large object.
103+ """
104+
105+ def __init__(self, *args, **kwargs):
106+ self.block_size = kwargs.pop('block_size', 1 << 16)
107+ super(LargeObjectField, self).__init__(*args, **kwargs)
108+
109+ def db_type(self, connection):
110+ """Returns the database column data type for LargeObjectField."""
111+ # oid is the column type postgres uses to reference a large object
112+ return 'oid'
113+
114+ def contribute_to_class(self, cls, name):
115+ """Set the descriptor for the large object."""
116+ super(LargeObjectField, self).contribute_to_class(cls, name)
117+ setattr(cls, self.name, LargeObjectDescriptor(self))
118+
119+ def get_db_prep_value(self, value, connection=None, prepared=False):
120+ """python -> db: `oid` value"""
121+ if value is None:
122+ return None
123+ if isinstance(value, LargeObjectFile):
124+ if value.oid > 0:
125+ return value.oid
126+ raise AssertionError(
127+ "LargeObjectFile's oid must be greater than 0.")
128+ raise AssertionError(
129+ "Invalid LargeObjectField value (expected LargeObjectFile): '%s'"
130+ % repr(value))
131+
132+ def to_python(self, value):
133+ """db -> python: `LargeObjectFile`"""
134+ if value is None:
135+ return None
136+ elif isinstance(value, LargeObjectFile):
137+ return value
138+ elif isinstance(value, (int, long)):
139+ return LargeObjectFile(value, self, self.model, self.block_size)
140+ raise AssertionError(
141+ "Invalid LargeObjectField value (expected integer): '%s'"
142+ % repr(value))
143
144=== modified file 'src/maasserver/tests/models.py'
145--- src/maasserver/tests/models.py 2014-07-08 07:52:05 +0000
146+++ src/maasserver/tests/models.py 2014-08-07 18:42:43 +0000
147@@ -25,6 +25,7 @@
148 )
149 from maasserver.fields import (
150 JSONObjectField,
151+ LargeObjectField,
152 MAASIPAddressField,
153 XMLField,
154 )
155@@ -73,3 +74,8 @@
156
157 class MAASIPAddressFieldModel(Model):
158 ip_address = MAASIPAddressField()
159+
160+
161+class LargeObjectFieldModel(Model):
162+ name = CharField(max_length=255, unique=False)
163+ large_object = LargeObjectField(block_size=10)
164
165=== modified file 'src/maasserver/tests/test_api_node.py'
166--- src/maasserver/tests/test_api_node.py 2014-08-05 10:13:38 +0000
167+++ src/maasserver/tests/test_api_node.py 2014-08-07 18:42:43 +0000
168@@ -18,7 +18,6 @@
169 from cStringIO import StringIO
170 import httplib
171 import json
172-from netaddr import IPAddress
173 import sys
174
175 import bson
176@@ -53,6 +52,7 @@
177 NodeUserData,
178 )
179 from metadataserver.nodeinituser import get_node_init_user
180+from netaddr import IPAddress
181 from provisioningserver.utils import map_enum
182
183
184
185=== modified file 'src/maasserver/tests/test_fields.py'
186--- src/maasserver/tests/test_fields.py 2014-07-16 14:12:13 +0000
187+++ src/maasserver/tests/test_fields.py 2014-08-07 18:42:43 +0000
188@@ -27,6 +27,8 @@
189 from maasserver.enum import NODEGROUPINTERFACE_MANAGEMENT
190 from maasserver.fields import (
191 EditableBinaryField,
192+ LargeObjectField,
193+ LargeObjectFile,
194 MAC,
195 NodeGroupFormField,
196 register_mac_type,
197@@ -42,10 +44,13 @@
198 from maasserver.testing.testcase import MAASServerTestCase
199 from maasserver.tests.models import (
200 JSONFieldModel,
201+ LargeObjectFieldModel,
202 MAASIPAddressFieldModel,
203 XMLFieldModel,
204 )
205 from maastesting.djangotestcase import TestModelMixin
206+from maastesting.matchers import MockCalledOnceWith
207+from psycopg2 import OperationalError
208 from psycopg2.extensions import ISQLQuote
209
210
211@@ -401,3 +406,109 @@
212 results = MAASIPAddressFieldModel.objects.filter(
213 ip_address__lte='192.0.2.100')
214 self.assertItemsEqual([ip_object], results)
215+
216+
217+class TestLargeObjectField(TestModelMixin, MAASServerTestCase):
218+
219+ app = 'maasserver.tests'
220+
221+ def test_stores_data(self):
222+ data = factory.make_string()
223+ test_name = factory.make_name('name')
224+ test_instance = LargeObjectFieldModel(name=test_name)
225+ large_object = LargeObjectFile()
226+ with large_object.open('wb') as stream:
227+ stream.write(data)
228+ test_instance.large_object = large_object
229+ test_instance.save()
230+ test_instance = LargeObjectFieldModel.objects.get(name=test_name)
231+ with test_instance.large_object.open('rb') as stream:
232+ saved_data = stream.read()
233+ self.assertEqual(data, saved_data)
234+
235+ def test_with_exit_calls_close(self):
236+ data = factory.make_string()
237+ large_object = LargeObjectFile()
238+ with large_object.open('wb') as stream:
239+ self.addCleanup(large_object.close)
240+ mock_close = self.patch(large_object, 'close')
241+ stream.write(data)
242+ self.assertThat(mock_close, MockCalledOnceWith())
243+
244+ def test_unlink(self):
245+ data = factory.make_string()
246+ large_object = LargeObjectFile()
247+ with large_object.open('wb') as stream:
248+ stream.write(data)
249+ oid = large_object.oid
250+ large_object.unlink()
251+ self.assertEqual(0, large_object.oid)
252+ self.assertRaises(
253+ OperationalError,
254+ connection.connection.lobject, oid)
255+
256+ def test_interates_on_block_size(self):
257+ # String size is multiple of block_size in the testing model
258+ data = factory.make_string(10 * 2)
259+ test_name = factory.make_name('name')
260+ test_instance = LargeObjectFieldModel(name=test_name)
261+ large_object = LargeObjectFile()
262+ with large_object.open('wb') as stream:
263+ stream.write(data)
264+ test_instance.large_object = large_object
265+ test_instance.save()
266+ test_instance = LargeObjectFieldModel.objects.get(name=test_name)
267+ with test_instance.large_object.open('rb') as stream:
268+ offset = 0
269+ for block in stream:
270+ self.assertEqual(data[offset:offset + 10], block)
271+ offset += 10
272+
273+ def test_get_db_prep_value_returns_None_when_value_None(self):
274+ field = LargeObjectField()
275+ self.assertEqual(None, field.get_db_prep_value(None))
276+
277+ def test_get_db_prep_value_returns_oid_when_value_LargeObjectFile(self):
278+ oid = randint(1, 100)
279+ field = LargeObjectField()
280+ obj_file = LargeObjectFile()
281+ obj_file.oid = oid
282+ self.assertEqual(oid, field.get_db_prep_value(obj_file))
283+
284+ def test_get_db_prep_value_raises_error_when_oid_less_than_zero(self):
285+ oid = randint(-100, 0)
286+ field = LargeObjectField()
287+ obj_file = LargeObjectFile()
288+ obj_file.oid = oid
289+ self.assertRaises(AssertionError, field.get_db_prep_value, obj_file)
290+
291+ def test_get_db_prep_value_raises_error_when_not_LargeObjectFile(self):
292+ field = LargeObjectField()
293+ self.assertRaises(
294+ AssertionError, field.get_db_prep_value, factory.make_string())
295+
296+ def test_to_python_returns_None_when_value_None(self):
297+ field = LargeObjectField()
298+ self.assertEqual(None, field.to_python(None))
299+
300+ def test_to_python_returns_value_when_value_LargeObjectFile(self):
301+ field = LargeObjectField()
302+ obj_file = LargeObjectFile()
303+ self.assertEqual(obj_file, field.to_python(obj_file))
304+
305+ def test_to_python_returns_LargeObjectFile_when_value_int(self):
306+ oid = randint(1, 100)
307+ field = LargeObjectField()
308+ obj_file = field.to_python(oid)
309+ self.assertEqual(oid, obj_file.oid)
310+
311+ def test_to_python_returns_LargeObjectFile_when_value_long(self):
312+ oid = long(randint(1, 100))
313+ field = LargeObjectField()
314+ obj_file = field.to_python(oid)
315+ self.assertEqual(oid, obj_file.oid)
316+
317+ def test_to_python_raises_error_when_not_valid_type(self):
318+ field = LargeObjectField()
319+ self.assertRaises(
320+ AssertionError, field.to_python, factory.make_string())