Merge lp:~rvb/maas/file-delete-1.2 into lp:maas/1.2

[Raphaël Badin]

Backport revisions 1435, 1436 and 1437.

[Raphaël Badin (rvb)]

review: Approve

=== modified file 'src/maasserver/api.py'
--- src/maasserver/api.py	2013-02-14 11:42:56 +0000
+++ src/maasserver/api.py	2013-02-14 15:08:26 +0000
@@ -61,6 +61,7 @@
     "api_doc",
     "api_doc_title",
     "BootImagesHandler",
+    "FileHandler",
     "FilesHandler",
     "get_oauth_token",
     "MaasHandler",
@@ -101,6 +102,7 @@
 from django.db.utils import DatabaseError
 from django.forms.models import model_to_dict
 from django.http import (
+    Http404,
     HttpResponse,
     HttpResponseBadRequest,
     QueryDict,
@@ -110,6 +112,7 @@
     render_to_response,
     )
 from django.template import RequestContext
+from django.utils.http import urlquote_plus
 from docutils import core
 from formencode import validators
 from formencode.validators import Invalid
@@ -173,10 +176,12 @@
     strip_domain,
     )
 from maasserver.utils.orm import get_one
+from piston.emitters import JSONEmitter
 from piston.handler import (
     AnonymousBaseHandler,
     BaseHandler,
     HandlerMetaClass,
+    typemapper,
     )
 from piston.models import Token
 from piston.resource import Resource
@@ -1052,6 +1057,61 @@
         return ('files_handler', [])
 
 
+# DISPLAYED_FILES_FIELDS_OBJECT is the list of fields used when dumping
+# lists of FileStorage objects.
+DISPLAYED_FILES_FIELDS = ('filename', )
+# DISPLAYED_FILES_FIELDS_OBJECT is the list of fields used when dumping
+# individual FileStorage objects.
+DISPLAYED_FILES_FIELDS_OBJECT = DISPLAYED_FILES_FIELDS + ('content', )
+
+
+class FileHandler(OperationsHandler):
+    """Manage a FileStorage object.
+
+    The file is identified by its filename.
+    """
+    model = FileStorage
+    fields = DISPLAYED_FILES_FIELDS
+    create = update = None
+
+    def read(self, request, filename):
+        """GET a FileStorage object as a json object.
+
+        The 'content' of the file is base64-encoded."""
+        # 'content' is a BinaryField, its 'value' is a base64-encoded version
+        # of its value.
+        stored_files = FileStorage.objects.filter(filename=filename).values()
+        # filename is a 'unique' field, we can have either 0 or 1 objects in
+        # 'stored_files'.
+        if len(stored_files) == 0:
+            raise Http404
+        stored_file = stored_files[0]
+        # Emit the json for this object manually because, no matter what the
+        # piston documentation says, once an type is associated with a list
+        # of fields by piston's typemapper mechanism, there is no way to
+        # override that in a specific handler with 'fields' or 'exclude'.
+        emitter = JSONEmitter(
+            stored_file, typemapper, None, DISPLAYED_FILES_FIELDS_OBJECT)
+        stream = emitter.render(request)
+        return HttpResponse(
+            stream, mimetype='application/json; charset=utf-8',
+            status=httplib.OK)
+
+    @operation(idempotent=False)
+    def delete(self, request, filename):
+        """Delete a FileStorage object."""
+        stored_file = get_object_or_404(FileStorage, filename=filename)
+        stored_file.delete()
+        return rc.DELETED
+
+    @classmethod
+    def resource_uri(cls, stored_file=None):
+        filename = "filename"
+        if stored_file is not None:
+            filename = urlquote_plus(stored_file.filename)
+        return ('file_handler', (filename, ))
+
+
 class FilesHandler(OperationsHandler):
     """File management operations."""
     create = read = update = delete = None
@@ -1084,6 +1144,23 @@
         FileStorage.objects.save_file(filename, uploaded_file)
         return HttpResponse('', status=httplib.CREATED)
 
+    @operation(idempotent=True)
+    def list(self, request):
+        """List the files from the file storage.
+
+        The returned files are ordered by file name and the content is
+        excluded.
+
+        :param prefix: Optional prefix used to filter out the returned files.
+        :type prefix: string
+        """
+        prefix = request.GET.get("prefix", None)
+        files = FileStorage.objects.all()
+        if prefix is not None:
+            files = files.filter(filename__startswith=prefix)
+        files = files.order_by('filename')
+        return files
+
     @classmethod
     def resource_uri(cls, *args, **kwargs):
         return ('files_handler', [])

=== modified file 'src/maasserver/tests/test_api.py'
--- src/maasserver/tests/test_api.py	2013-01-30 15:55:49 +0000
+++ src/maasserver/tests/test_api.py	2013-02-14 15:08:26 +0000
@@ -16,7 +16,10 @@
     ABCMeta,
     abstractproperty,
     )
-from base64 import b64encode
+from base64 import (
+    b64decode,
+    b64encode,
+    )
 from collections import namedtuple
 from cStringIO import StringIO
 from datetime import (
@@ -46,6 +49,7 @@
     )
 from django.http import QueryDict
 from django.test.client import RequestFactory
+from django.utils.http import urlquote_plus
 from fixtures import (
     EnvironmentVariableFixture,
     Fixture,
@@ -86,6 +90,7 @@
     BootImage,
     Config,
     DHCPLease,
+    FileStorage,
     MACAddress,
     Node,
     NodeGroup,
@@ -124,6 +129,7 @@
 from maastesting.djangotestcase import TransactionTestCase
 from maastesting.fakemethod import FakeMethod
 from maastesting.matchers import ContainsAll
+from maastesting.utils import sample_binary_data
 from metadataserver.models import (
     NodeKey,
     NodeUserData,
@@ -2703,6 +2709,29 @@
         self.assertEqual(httplib.OK, response.status_code)
         self.assertEqual(b"give me rope", response.content)
 
+    def test_anon_cannot_list_files(self):
+        factory.make_file_storage(
+             filename="filename", content=b"test content")
+        response = self.make_API_GET_request("list")
+        # The 'list' operation is not available to anon users.
+        self.assertEqual(httplib.BAD_REQUEST, response.status_code)
+
+    def test_anon_cannot_get_file(self):
+        filename = factory.make_name("file")
+        factory.make_file_storage(
+            filename=filename, content=b"test file content")
+        response = self.client.get(
+            reverse('file_handler', args=[filename]))
+        self.assertEqual(httplib.UNAUTHORIZED, response.status_code)
+
+    def test_anon_cannot_delete_file(self):
+        filename = factory.make_name('file')
+        factory.make_file_storage(
+            filename=filename, content=b"test content")
+        response = self.client.delete(
+            reverse('file_handler', args=[filename]))
+        self.assertEqual(httplib.UNAUTHORIZED, response.status_code)
+
 
 class FileStorageAPITest(FileStorageAPITestMixin, APITestCase):
 
@@ -2788,6 +2817,72 @@
         self.assertIn('text/plain', response['Content-Type'])
         self.assertEqual("File not found", response.content)
 
+    def test_list_files_returns_ordered_list(self):
+        filenames = ["myfiles/a", "myfiles/z", "myfiles/b"]
+        for filename in filenames:
+            factory.make_file_storage(
+                filename=filename, content=b"test content")
+        response = self.make_API_GET_request("list")
+        self.assertEqual(httplib.OK, response.status_code)
+        parsed_results = json.loads(response.content)
+        filenames = [result['filename'] for result in parsed_results]
+        self.assertEqual(sorted(filenames), filenames)
+
+    def test_list_files_lists_files_with_prefix(self):
+        filenames_with_prefix = ["prefix-file1", "prefix-file2"]
+        filenames = filenames_with_prefix + ["otherfile", "otherfile2"]
+        for filename in filenames:
+            factory.make_file_storage(
+                filename=filename, content=b"test content")
+        response = self.client.get(
+            self.get_uri('files/'), {"op": "list", "prefix": "prefix-"})
+        self.assertEqual(httplib.OK, response.status_code)
+        parsed_results = json.loads(response.content)
+        filenames = [result['filename'] for result in parsed_results]
+        self.assertItemsEqual(filenames_with_prefix, filenames)
+
+    def test_list_files_does_not_include_file_content(self):
+        factory.make_file_storage(
+             filename="filename", content=b"test content")
+        response = self.make_API_GET_request("list")
+        parsed_results = json.loads(response.content)
+        self.assertNotIn('content', parsed_results[0].keys())
+
+    def test_files_resource_uri_are_url_escaped(self):
+        filename = "&*!c/%//filename/"
+        factory.make_file_storage(
+             filename=filename, content=b"test content")
+        response = self.make_API_GET_request("list")
+        parsed_results = json.loads(response.content)
+        resource_uri = parsed_results[0]['resource_uri']
+        resource_uri_elements = resource_uri.split('/')
+        # The url-escaped name of the file is part of the resource uri.
+        self.assertIn(urlquote_plus(filename), resource_uri_elements)
+
+    def test_get_file_returns_file_object_with_content_base64_encoded(self):
+        filename = factory.make_name("file")
+        content = sample_binary_data
+        factory.make_file_storage(filename=filename, content=content)
+        response = self.client.get(
+            reverse('file_handler', args=[filename]))
+        parsed_result = json.loads(response.content)
+        self.assertEqual(
+            (filename, content),
+            (
+                parsed_result['filename'],
+                b64decode(parsed_result['content'])
+            ))
+
+    def test_delete_file_deletes_file(self):
+        filename = factory.make_name('file')
+        factory.make_file_storage(
+            filename=filename, content=b"test content")
+        response = self.client.delete(
+            reverse('file_handler', args=[filename]))
+        self.assertEqual(httplib.NO_CONTENT, response.status_code)
+        files = FileStorage.objects.filter(filename=filename)
+        self.assertEqual([], list(files))
+
 
 class TestTagAPI(APITestCase):
     """Tests for /api/1.0/tags/<tagname>/."""

=== modified file 'src/maasserver/urls_api.py'
--- src/maasserver/urls_api.py	2012-12-20 14:50:18 +0000
+++ src/maasserver/urls_api.py	2013-02-14 15:08:26 +0000
@@ -22,6 +22,7 @@
     api_doc,
     BootImagesHandler,
     describe,
+    FileHandler,
     FilesHandler,
     MaasHandler,
     NodeGroupHandler,
@@ -42,6 +43,7 @@
 
 account_handler = RestrictedResource(AccountHandler, authentication=api_auth)
 files_handler = RestrictedResource(FilesHandler, authentication=api_auth)
+file_handler = RestrictedResource(FileHandler, authentication=api_auth)
 node_handler = RestrictedResource(NodeHandler, authentication=api_auth)
 nodes_handler = RestrictedResource(NodesHandler, authentication=api_auth)
 node_mac_handler = RestrictedResource(NodeMacHandler, authentication=api_auth)
@@ -94,6 +96,7 @@
     url(r'nodegroups/(?P<uuid>[^/]+)/interfaces/(?P<interface>[^/]+)/$',
         nodegroupinterface_handler, name='nodegroupinterface_handler'),
     url(r'files/$', files_handler, name='files_handler'),
+    url(r'files/(?P<filename>[^/]+)/$', file_handler, name='file_handler'),
     url(r'account/$', account_handler, name='account_handler'),
     url(r'boot-images/$', boot_images_handler, name='boot_images_handler'),
     url(r'tags/(?P<name>[\w\-]+)/$', tag_handler, name='tag_handler'),

=== modified file 'src/maastesting/utils.py'
--- src/maastesting/utils.py	2012-07-16 10:19:46 +0000
+++ src/maastesting/utils.py	2013-02-14 15:08:26 +0000
@@ -17,8 +17,10 @@
     "get_write_time",
     "preexec_fn",
     "retries",
+    "sample_binary_data",
     ]
 
+import codecs
 import os
 import re
 import signal
@@ -90,3 +92,16 @@
             sleep(min(delay, end - now))
         else:
             break
+
+
+# Some horrible binary data that could never, ever, under any encoding
+# known to man(1) survive mis-interpretation as text.
+#
+# The data contains a nul byte to trip up accidental string termination.
+# Switch the bytes of the byte-order mark around and by design you get
+# an invalid codepoint; put a byte with the high bit set between bytes
+# that have it cleared, and you have a guaranteed non-UTF-8 sequence.
+#
+# (1) Provided, of course, that man know only about ASCII and
+# UTF.
+sample_binary_data = codecs.BOM64_LE + codecs.BOM64_BE + b'\x00\xff\x00'